mindspore 2.3.0__cp310-cp310-win_amd64.whl → 2.4.1__cp310-cp310-win_amd64.whl

mindspore/communication/management.py

@@ -102,7 +102,8 @@ def _set_envs():
     os.environ["RANK_ID"] = str(get_rank())
     if os.getenv("RANK_SIZE") is None:
         os.environ["RANK_SIZE"] = str(get_group_size())
-    os.environ["DEVICE_ID"] = str(get_local_rank())
+    if os.getenv("DEVICE_ID") is None:
+        os.environ["DEVICE_ID"] = str(get_local_rank())
 
 
 def init(backend_name=None):
@@ -140,7 +141,7 @@ def init(backend_name=None):
             For Ascend/GPU/CPU devices, it is recommended to use the msrun startup method
             without any third-party or configuration file dependencies.
             Please see the `msrun start up
-            <https://www.mindspore.cn/tutorials/experts/zh-CN/master/parallel/msrun_launcher.html>`_
+            <https://www.mindspore.cn/docs/zh-CN/master/model_train/parallel/msrun_launcher.html>`_
             for more details.
 
         >>> from mindspore.communication import init
@@ -165,6 +166,7 @@ def init(backend_name=None):
     if os.getenv("MS_ROLE") == "MS_SCHED":
         backend_name = "mccl"
 
+    _set_elegant_exit_handle()
     if backend_name == "hccl":
         if _is_ps_mode():
             # Use MindSpore cluster to build network for Parameter Server training.
@@ -173,7 +175,6 @@ def init(backend_name=None):
                 raise RuntimeError("Parameter server and scheduler should use 'CPU' as backend instead of 'Ascend'")
             if _get_ps_context("worker_num") == 1:
                 GlobalComm.INITED = True
-                _set_elegant_exit_handle()
                 return
         if device_target != "Ascend":
             raise RuntimeError("For 'init', the argument 'backend_name' should be '{}' to init '{}', "
@@ -203,7 +204,6 @@ def init(backend_name=None):
                            "but got 'backend_name' : {}".format(backend_name))
 
     GlobalComm.INITED = True
-    _set_elegant_exit_handle()
     _set_envs()
 
 
@@ -227,7 +227,7 @@ def release():
             For Ascend/GPU/CPU devices, it is recommended to use the msrun startup method
             without any third-party or configuration file dependencies.
             Please see the `msrun start up
-            <https://www.mindspore.cn/tutorials/experts/zh-CN/master/parallel/msrun_launcher.html>`_
+            <https://www.mindspore.cn/docs/zh-CN/master/model_train/parallel/msrun_launcher.html>`_
             for more details.
 
         >>> from mindspore.communication import init, release
@@ -266,7 +266,7 @@ def get_rank(group=GlobalComm.WORLD_COMM_GROUP):
             For Ascend/GPU/CPU devices, it is recommended to use the msrun startup method
             without any third-party or configuration file dependencies.
             Please see the `msrun start up
-            <https://www.mindspore.cn/tutorials/experts/zh-CN/master/parallel/msrun_launcher.html>`_
+            <https://www.mindspore.cn/docs/zh-CN/master/model_train/parallel/msrun_launcher.html>`_
             for more details.
 
         >>> from mindspore.communication import init, get_rank
@@ -311,7 +311,7 @@ def get_local_rank(group=GlobalComm.WORLD_COMM_GROUP):
             For Ascend/GPU/CPU devices, it is recommended to use the msrun startup method
             without any third-party or configuration file dependencies.
             Please see the `msrun start up
-            <https://www.mindspore.cn/tutorials/experts/zh-CN/master/parallel/msrun_launcher.html>`_
+            <https://www.mindspore.cn/docs/zh-CN/master/model_train/parallel/msrun_launcher.html>`_
             for more details.
 
         >>> import mindspore as ms
@@ -359,7 +359,7 @@ def get_group_size(group=GlobalComm.WORLD_COMM_GROUP):
             For Ascend/GPU/CPU devices, it is recommended to use the msrun startup method
             without any third-party or configuration file dependencies.
             Please see the `msrun start up
-            <https://www.mindspore.cn/tutorials/experts/zh-CN/master/parallel/msrun_launcher.html>`_
+            <https://www.mindspore.cn/docs/zh-CN/master/model_train/parallel/msrun_launcher.html>`_
             for more details.
 
         >>> import mindspore as ms
@@ -406,7 +406,7 @@ def get_local_rank_size(group=GlobalComm.WORLD_COMM_GROUP):
             For Ascend/GPU/CPU devices, it is recommended to use the msrun startup method
             without any third-party or configuration file dependencies.
             Please see the `msrun start up
-            <https://www.mindspore.cn/tutorials/experts/zh-CN/master/parallel/msrun_launcher.html>`_
+            <https://www.mindspore.cn/docs/zh-CN/master/model_train/parallel/msrun_launcher.html>`_
             for more details.
 
         >>> import mindspore as ms
@@ -456,7 +456,7 @@ def get_world_rank_from_group_rank(group, group_rank_id):
             For Ascend/GPU/CPU devices, it is recommended to use the msrun startup method
             without any third-party or configuration file dependencies.
             Please see the `msrun start up
-            <https://www.mindspore.cn/tutorials/experts/zh-CN/master/parallel/msrun_launcher.html>`_
+            <https://www.mindspore.cn/docs/zh-CN/master/model_train/parallel/msrun_launcher.html>`_
             for more details.
 
         >>> import mindspore as ms
@@ -510,7 +510,7 @@ def get_group_rank_from_world_rank(world_rank_id, group):
             For Ascend/GPU/CPU devices, it is recommended to use the msrun startup method
             without any third-party or configuration file dependencies.
             Please see the `msrun start up
-            <https://www.mindspore.cn/tutorials/experts/zh-CN/master/parallel/msrun_launcher.html>`_
+            <https://www.mindspore.cn/docs/zh-CN/master/model_train/parallel/msrun_launcher.html>`_
             for more details.
 
         >>> import mindspore as ms
@@ -561,7 +561,7 @@ def create_group(group, rank_ids):
             For Ascend/GPU/CPU devices, it is recommended to use the msrun startup method
             without any third-party or configuration file dependencies.
             Please see the `msrun start up
-            <https://www.mindspore.cn/tutorials/experts/zh-CN/master/parallel/msrun_launcher.html>`_
+            <https://www.mindspore.cn/docs/zh-CN/master/model_train/parallel/msrun_launcher.html>`_
             for more details.
 
         >>> import mindspore as ms
@@ -609,7 +609,7 @@ def destroy_group(group):
             For Ascend/GPU/CPU devices, it is recommended to use the msrun startup method
             without any third-party or configuration file dependencies.
             Please see the `msrun start up
-            <https://www.mindspore.cn/tutorials/experts/zh-CN/master/parallel/msrun_launcher.html>`_
+            <https://www.mindspore.cn/docs/zh-CN/master/model_train/parallel/msrun_launcher.html>`_
             for more details.
 
         >>> import mindspore as ms
@@ -656,7 +656,7 @@ def get_process_group_ranks(group=GlobalComm.WORLD_COMM_GROUP):
             without any third-party or configuration file dependencies.
 
             Please see the `msrun start up
-            <https://www.mindspore.cn/tutorials/experts/zh-CN/master/parallel/msrun_launcher.html>`_
+            <https://www.mindspore.cn/docs/zh-CN/master/model_train/parallel/msrun_launcher.html>`_
             for more details.
 
             This example should be run with 4 devices.

mindspore/context.py

@@ -35,6 +35,7 @@ from mindspore.parallel._ps_context import _set_ps_context, _get_ps_context, _re
     _need_reset_device_target_for_ps
 from mindspore.parallel._offload_context import _set_offload_context, _get_offload_context
 from mindspore.hal.device import is_initialized
+from mindspore.common import api
 
 __all__ = ['GRAPH_MODE', 'PYNATIVE_MODE', 'STRICT', 'COMPATIBLE', 'LAX', 'set_context', 'get_context',
            'set_auto_parallel_context', 'get_auto_parallel_context', 'reset_auto_parallel_context', 'ParallelMode',
@@ -68,7 +69,7 @@ def _make_directory(path):
     if not os.path.exists(path):
         logger.debug("The directory(%s) doesn't exist, will create it", path)
         try:
-            os.makedirs(path)
+            os.makedirs(path, mode=0o700)
         except FileExistsError:
             logger.debug("The directory(%s) already exist.", path)
         except PermissionError as e:
@@ -168,7 +169,10 @@ class _Context:
         self._support_binary = False
         self.enable_compile_cache = None
         self._mode = PYNATIVE_MODE
-        self._jit_config = {}
+        self.aoe_config = {}
+        self.jit_config = {}
+        self.ascend_config = {}
+        self.gpu_config = {}
 
     def __getattribute__(self, attr):
         value = object.__getattribute__(self, attr)
@@ -188,7 +192,7 @@ class _Context:
 
     def get_jit_config(self):
         """Get current jit_config."""
-        return self._jit_config
+        return self.jit_config
 
     def set_mode(self, mode):
         """
@@ -248,6 +252,16 @@ class _Context:
         else:
             self.set_param(ms_ctx_param.memory_optimize_level, 1)
 
+    def set_exec_order(self, exec_order):
+        """
+        The execution order mode, support "bfs", "dfs", "gpto".
+        """
+        exec_order_modes = ["bfs", "dfs", "gpto"]
+        if exec_order not in exec_order_modes:
+            raise ValueError(f"For 'context.set_context', the argument 'exec_order' must be one of "
+                             f"{exec_order_modes}, but got {exec_order}.")
+        self.set_param(ms_ctx_param.exec_order, exec_order)
+
     def set_memory_offload(self, memory_offload):
         """
         Enable memory offload or not, support "ON", "OFF".
@@ -277,6 +291,29 @@ class _Context:
                              f"{deterministic_options}, but got {deterministic}.")
         self.set_param(ms_ctx_param.deterministic, deterministic)
 
+        hccl_deterministic = os.getenv("HCCL_DETERMINISTIC")
+        te_parallel_compiler = os.getenv("TE_PARALLEL_COMPILER")
+        if deterministic == "ON":
+            if hccl_deterministic and hccl_deterministic != "true":
+                logger.warning(f"Environment 'HCCL_DETERMINISTIC' should be 'true' when set deterministic='ON', but "
+                               f"got '{hccl_deterministic}'. 'HCCL_DETERMINISTIC' will be set to 'true'.")
+            if te_parallel_compiler and te_parallel_compiler != "1":
+                logger.warning(f"Environment 'TE_PARALLEL_COMPILER' should be '1' when set deterministic='ON', but "
+                               f"got '{te_parallel_compiler}'. 'TE_PARALLEL_COMPILER' will be set to '1'.")
+            os.environ["HCCL_DETERMINISTIC"] = "true"
+            os.environ["TE_PARALLEL_COMPILER"] = "1"
+        if deterministic == "OFF":
+            if hccl_deterministic and hccl_deterministic != "false":
+                logger.warning(f"Environment 'HCCL_DETERMINISTIC' should not be set or be 'false' when set "
+                               f"deterministic='OFF', but got '{hccl_deterministic}'. 'HCCL_DETERMINISTIC' "
+                               f"will be unset.")
+                del os.environ["HCCL_DETERMINISTIC"]
+            if te_parallel_compiler and te_parallel_compiler != "0":
+                logger.warning(f"Environment 'TE_PARALLEL_COMPILER' should not be set or be '0' when set "
+                               f"deterministic='OFF', but got '{te_parallel_compiler}'. 'TE_PARALLEL_COMPILER' "
+                               f"will be unset.")
+                del os.environ["TE_PARALLEL_COMPILER"]
+
     def set_ascend_config(self, ascend_config):
         """
         Enable ascend config.
@@ -298,6 +335,8 @@ class _Context:
                 - parallel_speed_up_json_path(Union[str, None]): The path to the parallel speed up json file.
                   If its value is None or '', it does not take effect. Default None.
                 - host_scheduling_max_threshold(int): The host scheduling max threshold.
+                - hccl_watchdog (bool): Enable a thread to monitor the failure of collective communication.
+                  Default: ``True`` .
         """
         ascend_cfg_modes = {
             'precision_mode': ["force_fp16", "allow_fp32_to_fp16", "allow_mix_precision", "must_keep_origin_dtype",
@@ -316,6 +355,7 @@ class _Context:
             'save_checkpoint_steps': (int,),
             'need_ckpt': (bool,),
             'last_triggered_step': (int,),
+            'hccl_watchdog': (bool,),
             'topo_order': (dict,),
             'op_debug_option': (str, None),
         }
@@ -335,6 +375,7 @@ class _Context:
             'save_checkpoint_steps': self._set_save_checkpoint_steps,
             'need_ckpt': self._set_need_ckpt,
             'last_triggered_step': self._set_last_triggered_step,
+            'hccl_watchdog': self._set_hccl_watchdog,
             'topo_order': self._set_topo_order
         }
         ascend_cfg_set = tuple(ascend_cfg_modes.keys())
@@ -351,6 +392,7 @@ class _Context:
                                 f"{supported_modes}, but got {type(ascend_value)}.")
             cfg_setter = ascend_cfg_setters.get(ascend_key)
             cfg_setter(ascend_value)
+        self.ascend_config = ascend_config
 
     def set_gpu_config(self, gpu_config):
         """
@@ -392,6 +434,7 @@ class _Context:
                 self.set_param(ms_ctx_param.conv_allow_tf32, gpu_config[gpu_key])
             if gpu_key == 'matmul_allow_tf32':
                 self.set_param(ms_ctx_param.matmul_allow_tf32, gpu_config[gpu_key])
+            self.gpu_config = gpu_config
 
     def set_jit_config(self, jit_config):
         """
@@ -410,12 +453,13 @@ class _Context:
                                  f"{jit_cfgs}, but got {jit_key}.")
             supported_value = jit_cfgs.get(jit_key)
             if jit_config[jit_key] not in supported_value:
-                raise ValueError(f"For 'jit_cfgs', the value of argument {jit_key} must be one of "
+                raise ValueError(f"For 'jit_config', the value of argument {jit_key} must be one of "
                                  f"{supported_value}, but got {jit_config[jit_key]}.")
-            self._jit_config = jit_config
             self.set_param(key_args_map[jit_key], jit_config[jit_key])
+        self.jit_config = jit_config
 
-        if 'infer_boost' in jit_config and jit_config['infer_boost'] == "on" and jit_config['jit_level'] != "O0":
+        jit_level = jit_config.get("jit_level", None)
+        if jit_config.get("infer_boost", None) == "on" and (jit_level == "O1" or jit_level == "O2"):
             raise ValueError(f"Only jit_level set O0 can set infer_boost to on.")
 
     def set_backend_policy(self, policy):
@@ -488,6 +532,7 @@ class _Context:
                                  f"{supported_value}, but got {aoe_config[aoe_config_key]}.")
             if aoe_config_key == 'job_type':
                 self.set_param(ms_ctx_param.aoe_job_type, aoe_config[aoe_config_key])
+        self.aoe_config = aoe_config
 
     def set_device_id(self, device_id):
         if device_id < 0 or device_id > 4095:
@@ -626,6 +671,7 @@ class _Context:
         'inter_op_parallel_num': set_inter_op_parallel_num,
         'runtime_num_threads': set_runtime_num_threads,
         'memory_optimize_level': set_memory_optimize_level,
+        'exec_order': set_exec_order,
         'op_timeout': set_op_timeout,
         'memory_offload': set_memory_offload,
         'deterministic': set_deterministic,
@@ -744,6 +790,12 @@ class _Context:
         options_str = json.dumps(topo_order)
         self.set_param(ms_ctx_param.topo_order, options_str)
 
+    def _set_hccl_watchdog(self, flag):
+        """set hccl watchdog"""
+        if not isinstance(flag, bool):
+            raise TypeError(f"For 'ascend_config', the type of 'hccl_watchdog' must be bool, but got {type(flag)}.")
+        self.set_param(ms_ctx_param.hccl_watchdog, flag)
+
     def _set_need_ckpt(self, need_ckpt):
         """Set need ckpt flag"""
         if not isinstance(need_ckpt, bool):
@@ -772,7 +824,7 @@ class _Context:
         """"Check and set speedup config for auto parallel."""
         if speedup_config_path is None or speedup_config_path == "":
             return
-        speedup_config_real_path = os.path.abspath(speedup_config_path)
+        speedup_config_real_path = os.path.realpath(speedup_config_path)
         if not os.path.exists(speedup_config_real_path):
             raise ValueError(f"For 'ascend_config', the path to parallel_speed_up_json: "
                              f"{speedup_config_real_path} does not exist, please check whether the "
@@ -790,10 +842,17 @@ class _Context:
                             "enable_begin_end_inline_opt": (ms_ctx_param.enable_begin_end_inline_opt, bool),
                             "enable_concat_eliminate_opt": (ms_ctx_param.enable_concat_eliminate_opt, bool),
                             "interleaved_layernorm_comm": (ms_ctx_param.interleaved_layernorm_comm, bool),
+                            "enable_allreduce_slice_to_reducescatter":
+                                (ms_ctx_param.enable_allreduce_slice_to_reducescatter, bool),
+                            "enable_interleave_split_concat_branch":
+                                (ms_ctx_param.enable_interleave_split_concat_branch, bool),
+                            "enable_offloading_packed_experts": (ms_ctx_param.enable_offloading_packed_experts, bool),
                             "compute_communicate_fusion_level":
                                 (ms_ctx_param.compute_communicate_fusion_level, int),
                             "enable_flash_attention_load_balance":
-                                (ms_ctx_param.enable_flash_attention_load_balance, bool)}
+                                (ms_ctx_param.enable_flash_attention_load_balance, bool),
+                            "dataset_broadcast_opt_level":
+                                (ms_ctx_param.dataset_broadcast_opt_level, int)}
             with open(speedup_config_real_path, 'r') as f:
                 speedup_config = json.load(f)
                 for key, value in speedup_config.items():
@@ -876,6 +935,7 @@ def set_auto_parallel_context(**kwargs):
                \                 strategy_ckpt_config
                \                 group_ckpt_save_file
                \                 auto_pipeline
+               \                 dump_local_norm
     ===========================  ===========================
 
     Args:
@@ -1027,6 +1087,9 @@ def set_auto_parallel_context(**kwargs):
         auto_pipeline (bool): Set the pipeline stage number to automatic. Its value will be selected between 1 and the
                         parameter `pipeline_stages`. This option requires the `parallel_mode` to be ``auto_parallel``
                         and the `search_mode` to be ``recursive_programming``. Default: ``False`` .
+        dump_local_norm (bool): Whether to dump local_norm value, when the `parallel_mode` is set to
+                        ``semi_auto_parallel`` or ``auto_parallel``.
+                        Default: ``False`` .
 
     Raises:
         ValueError: If input key is not attribute in auto parallel context.
@@ -1097,11 +1160,12 @@ def reset_auto_parallel_context():
     - strategy_ckpt_save_file: ''.
     - full_batch: False.
     - enable_parallel_optimizer: False.
-    - force_fp32_communication: False
+    - force_fp32_communication: False.
     - enable_alltoall: False.
     - pipeline_stages: 1.
     - pipeline_result_broadcast: False.
     - fusion_threshold: 64.
+    - dump_local_norm: False.
     - auto_pipeline: False.
 
     Examples:
@@ -1109,6 +1173,7 @@ def reset_auto_parallel_context():
         >>> ms.reset_auto_parallel_context()
     """
     _reset_auto_parallel_context()
+    api.ms_compile_cache.clear()
 
 
 @args_type_check(offload_config=dict)
@@ -1118,7 +1183,8 @@ def set_offload_context(offload_config):
 
     Note:
         The offload configuration is only used if the memory offload feature is enabled
-        via mindspore.set_context(memory_offload="ON").
+        via mindspore.set_context(memory_offload="ON"), and the memory_optimize_level must be set to O0. On the Ascend
+        hardware platform, the graph compilation level must be O0.
 
     Args:
         offload_config (dict): A dict contains the keys and values for setting the offload context
@@ -1311,6 +1377,8 @@ def set_context(**kwargs):
     |                         |  gpu_config                  |  GPU                       |
     |                         +------------------------------+----------------------------+
     |                         |  jit_config                  |  CPU/GPU/Ascend            |
+    |                         +------------------------------+----------------------------+
+    |                         |  exec_order                  |  Ascend                    |
     +-------------------------+------------------------------+----------------------------+
 
     Args:
@@ -1320,12 +1388,14 @@ def set_context(**kwargs):
             If device target is not set, the version of MindSpore package is used.
         max_device_memory (str): Set the maximum memory available for devices. The format is "xxGB".
             Default: ``" 1024GB"`` . The actual used memory size is the minimum of the available memory of the device
-            and max_device_memory. 'max_device_memory' should be set before the program runs.
+            and max_device_memory. 'max_device_memory' should be set before the program runs. When virtual memory is
+            enabled, a too small 'max_device_memory' will cause frequent defragmentation, affecting performance.
         variable_memory_max_size (str): This parameter is deprecated, and will be removed in a future version.
             Please use parameter 'max_device_memory' instead.
-        mempool_block_size (str): Set the size of the memory pool block in PyNative mode or jit level is 'O0'/'O1'
-            for devices. The format is "xxGB". Default: ``"1GB"`` . Minimum size is "1G". The actual used memory block
-            size is the minimum of the available memory of the device and mempool_block_size.
+        mempool_block_size (str): It takes effect when virtual memory is turned off, set the size of the memory pool
+            block for devices. The format is "xxGB". Default: ``"1GB"`` . Minimum size is "1G". The actual used memory
+            block size is the minimum of the available memory of the device and mempool_block_size. When there is
+            enough memory, the memory will be expanded by this value.
         op_timeout (int): Set the maximum duration of executing an operator in seconds.
             If the execution time exceeds this value, system will terminate the task.
             0 means endless wait. The defaults for AI Core and AICPU operators vary on different hardware.
@@ -1413,7 +1483,7 @@ def set_context(**kwargs):
             If enable_graph_kernel is set to ``True`` , acceleration can be enabled.
             For details of graph kernel fusion, please check
             `Enabling Graph Kernel Fusion
-            <https://www.mindspore.cn/tutorials/experts/en/master/optimize/graph_fusion_engine.html>`_.
+            <https://www.mindspore.cn/docs/en/master/model_train/optimize/graph_fusion_engine.html>`_.
         graph_kernel_flags (str):
             Optimization options of graph kernel fusion, and the priority is higher when it conflicts
             with enable_graph_kernel. Only for experienced users.
@@ -1438,6 +1508,11 @@ def set_context(**kwargs):
                 Be caution when using this level.
 
             - dump_as_text: dumps detail info as text files. Default: ``False`` .
+            - enable_cluster_ops: Add user-specified operator to the set of operators involved in fusion. For example,
+              by setting ``--enable_cluster_ops=MatMul``, MatMul operator can be included in the fusion process.
+            - enable_pass/disable_pass: Enable/disable user-specified custom fusion passes. See details in
+              `Custom Fusion Pass
+              <https://www.mindspore.cn/docs/en/master/model_train/custom_program/fusion_pass.html>`_.
 
         enable_reduce_precision (bool): Whether to enable precision reduction.
             If the operator does not support the user-specified precision, the precision will
@@ -1468,6 +1543,7 @@ def set_context(**kwargs):
             if enable_compile_cache is still set to ``True`` and the network scripts are not changed,
             the compile cache is loaded. Note that only limited automatic detection for the changes of
             python scripts is supported by now, which means that there is a correctness risk. Default: ``False`` .
+            Currently, do not support the graph which is larger than 2G after compiled.
             This is an experimental prototype that is subject to change and/or deletion.
         compile_cache_path (str): Path to save the compile cache. Default: ``"."``.
             If the specified directory does not exist, the system will automatically create the directory.
@@ -1477,7 +1553,8 @@ def set_context(**kwargs):
             which means use the default num.
         runtime_num_threads(int): The thread pool number of cpu kernel used in runtime,
             which must bigger than or equal to 0. Default value is ``30`` , if you run many processes at
-            the same time, you should set the value smaller to avoid thread contention.
+            the same time, you should set the value smaller to avoid thread contention. If set runtime_num_threads to 1,
+            the runtime asynchronous pipeline capability cannot be enabled, which may affect performance.
         disable_format_transform (bool): Whether to disable the automatic format transform function from NCHW to NHWC.
             When the network training performance of fp16 is worse than fp32, `disable_format_transform` can be set to
             ``True`` to try to improve training performance. Default: ``False`` .
@@ -1588,7 +1665,7 @@ def set_context(**kwargs):
                 `LazyInline <https://www.mindspore.cn/docs/en/master/api_python/mindspore/mindspore.lazy_inline.html>`
                 Default: False.
               - compute_communicate_fusion_level (int): Enable the fusion between compute and communicate.
-                Default: ``0``.
+                Default: ``0``. Note: This function must be used with Ascend Training Solution 24.0.RC2 or later.
 
                 - 0: Disable fusion.
 
@@ -1597,8 +1674,27 @@ def set_context(**kwargs):
                 - 2: Apply fusion to backward nodes.
 
                 - 3: Apply fusion to all nodes.
+              - dataset_broadcast_opt_level (int): Optimize the scenario that the dataset repeated reading. Only
+                support O0/O1 jit level. It doesn't work in O2 mode. Default: ``0``.
+
+                - 0: Disable this optimize.
+
+                - 1: Optimize dataset reader between pipeline stage.
+
+                - 2: Optimize dataset reader within pipeline stage.
+
+                - 3: Optimize dataset reader with all scenes.
               - bias_add_comm_swap (bool): Enable node execution order swap communication operators and add operators
                 if ``True``. Only 1-dimension bias node is supported. Default: ``False``.
+              - enable_allreduce_slice_to_reducescatter (bool): Enable allreduce optimization. In the scenario where
+                the batchmatmul model introduces allreduce in parallel, if the subsequent nodes are stridedslice
+                operator with model parallel, allreduce will be optimized as reducescatter according to the identified
+                patterns. Typical used in MoE module with groupwise alltoall. Default: ``False``.
+              - enable_interleave_split_concat_branch (bool): Enable communication computation parallel optimization
+                for branches formed by split and concat operators with ``enable_interleave`` attribute. It is typical
+                used in MoE parallel scenario. After splitting the input data, each slice of data is processed by the
+                MoE module, and then the branch results are concatenated. When the optimization is enable,
+                communication and computation will be executed in parallel between branches. Default: ``False``.
             - host_scheduling_max_threshold(int): The max threshold to control whether the dynamic shape process is
               used when run the static graph, the default value is 0. When the number of operations in the static graph
               is less than the max threshold, this graph will be executed in dynamic shape process. In large model
@@ -1698,12 +1794,13 @@ def set_context(**kwargs):
 
             - jit_level (str): Used to control the compilation optimization level. Default: ``""`` , The framework
               automatically selects the execution method based on product, Altas training product is O2, and all other
-              products are O0. The value range is as follows:
+              products are O0. In addition, The option of the dynamic shape must be O0 or O1, O2 is not supported.
+              The value range is as follows:
 
               - ``"O0"``: Except for optimizations that may affect functionality, all other optimizations are turned
                 off, adopt KernelByKernel execution mode.
               - ``"O1"``: Using commonly used optimizations and automatic operator fusion optimizations,
-                adopt KernelByKernel execution mode.
+                adopt KernelByKernel execution mode. This optimization level is experimental and is being improved.
               - ``"O2"``: Ultimate performance optimization, adopt Sink execution mode.
 
             - infer_boost (str): Used to control the infer mode. Default: ``"off"`` . The value range is as follows:
@@ -1711,6 +1808,18 @@ def set_context(**kwargs):
               - ``"on"``: Enable infer mode, get better infer performance.
               - ``"off"``: Disable infer mode, use forward to infer, performance is not good.
 
+        exec_order (str): Set the sorting method for operator execution in GRAPH_MODE Currently, only three sorting
+            methods are supported: bfs and gpto, and the default method is bfs.
+
+            - ``"bfs"``: The default sorting method, breadth priority, good communication masking, relatively good
+              performance.
+            - ``"dfs"``: An optional sorting method, depth-first sorting. The performance is relatively worse than that
+              of bfs execution order, but it occupies less memory. It is recommended to try dfs in scenarios where other
+              execution orders run out of memory (OOM).
+            - ``"gpto"``: An optional sorting method. This method combines multiple execution orders and selects a
+              method with relatively good performance. There may be some performance gains in scenarios with multiple
+              replicas running in parallel.
+
     Raises:
         ValueError: If input key is not an attribute in context.
 
@@ -1753,6 +1862,7 @@ def set_context(**kwargs):
         >>> ms.set_context(gpu_config={"conv_fprop_algo": "performance", "conv_allow_tf32": True,
         ...                "matmul_allow_tf32": True})
         >>> ms.set_context(jit_config={"jit_level": "O0"})
+        >>> ms.set_context(exec_order="gpto")
     """
     ctx = _context()
     # set device target first
@@ -1790,12 +1900,12 @@ def set_context(**kwargs):
             continue
         if not _check_target_specific_cfgs(device, key):
             continue
-        if hasattr(ctx, key):
-            setattr(ctx, key, value)
-            continue
         if key in ctx.setters:
             ctx.setters[key](ctx, value)
             continue
+        if hasattr(ctx, key):
+            setattr(ctx, key, value)
+            continue
         # enum variables beginning with '_' are for internal use
         if key in ms_ctx_param.__members__ and key[0] != '_':
             ctx.set_param(ms_ctx_param.__members__[key], value)

mindspore/dataset/__init__.py

@@ -21,7 +21,7 @@ Besides, this module provides APIs to sample data while loading.
 
 We can enable cache in most of the dataset with its key arguments 'cache'. Please notice that cache is not supported
 on Windows platform yet. Do not use it while loading and processing data on Windows. More introductions and limitations
-can refer `Single-Node Tensor Cache <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ .
+can refer `Single-Node Tensor Cache <https://www.mindspore.cn/docs/en/master/model_train/dataset/cache.html>`_ .
 
 Common imported modules in corresponding API examples are as follows:
 

mindspore/dataset/audio/__init__.py

@@ -43,7 +43,7 @@ The data transform operation can be executed in the data processing pipeline or
   `introduction to data processing pipeline <https://www.mindspore.cn/docs/en/master/api_python/
   mindspore.dataset.html#introduction-to-data-processing-pipeline>`_ .
 - Eager mode is more like a function call to process data. Examples refer to
-  `Lightweight Data Processing <https://www.mindspore.cn/tutorials/en/master/advanced/dataset/eager.html>`_ .
+  `Lightweight Data Processing <https://www.mindspore.cn/docs/en/master/model_train/dataset/eager.html>`_ .
 """
 from __future__ import absolute_import
 

mindspore/dataset/core/config.py

@@ -32,6 +32,8 @@ import mindspore._c_dataengine as cde
 from mindspore import log as logger
 from mindspore.dataset.core.validator_helpers import replace_none, type_check
 from mindspore.dataset.debug import DebugHook, PrintMetaDataHook
+from mindspore.dataset.core.validator_helpers import check_independent_mode
+
 
 __all__ = ['set_sending_batches', 'load', '_init_device_info',
            'set_seed', 'get_seed',
@@ -544,6 +546,8 @@ def set_enable_autotune(enable, filepath_prefix=None):
     if not isinstance(enable, bool):
         raise TypeError("enable must be of type bool.")
 
+    check_independent_mode("Dataset AutoTune", enable)
+
     save_autoconfig = bool(enable and filepath_prefix is not None)
 
     if filepath_prefix and not isinstance(filepath_prefix, str):
@@ -728,6 +732,9 @@ def set_auto_offload(offload):
     """
     if not isinstance(offload, bool):
         raise TypeError("offload must be a bool dtype")
+
+    check_independent_mode("Dataset Offload", offload)
+
     _config.set_auto_offload(offload)
 
 

mindspore/dataset/core/validator_helpers.py

@@ -766,6 +766,13 @@ def check_dict(data, key_type, value_type, param_name):
                                 .format(key, param_name, value_type, type(value)))
 
 
+def check_independent_mode(feature_name, condition=True):
+    # todo in Dataset Independent mode
+    independent_process_env = os.getenv("MS_INDEPENDENT_DATASET", None)
+    if condition and independent_process_env and independent_process_env.strip() in ['True', 'true']:
+        raise RuntimeError(f"{feature_name} is not supported in Dataset Independent mode.")
+
+
 def check_feature_shape(data, shape, param_name):
     if isinstance(data, dict):
         for key, value in data.items():

mindspore/dataset/engine/cache_client.py

@@ -27,7 +27,7 @@ class DatasetCache:
     A client to interface with tensor caching service.
 
     For details, please check
-    `Tutorial <https://www.mindspore.cn/tutorials/experts/en/master/dataset/cache.html>`_ .
+    `Tutorial <https://www.mindspore.cn/docs/en/master/model_train/dataset/cache.html>`_ .
 
     Args:
         session_id (int): A user assigned session id for the current pipeline.