mindspore 2.2.14__cp39-cp39-win_amd64.whl → 2.3.0__cp39-cp39-win_amd64.whl

mindspore/parallel/_utils.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Huawei Technologies Co., Ltd
+# Copyright 2023-2024 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,6 +27,8 @@ from mindspore.common.seed import get_seed
 from mindspore._c_expression import GraphExecutor_
 from mindspore.parallel._tensor import _load_tensor_by_layout
 
+SUPPORTED_TUPLE_IN_TUPLE_STRATEGY = ["GroupedMatmul", "FusedInferAttentionScore"]
+
 
 def _get_parallel_mode():
     """Get parallel mode."""
@@ -92,6 +94,18 @@ def _need_to_full():
 
 def _slice_parameter(parameter, phase, layout):
     """Slice python parameter obj according to the layout."""
+    is_train_phase = phase.startswith('train')
+    is_prefill_phase = phase.startswith('prefill')
+    if layout is not None and parameter.from_ckpt and not is_train_phase:
+        is_opt_shard_group = layout[5]
+        if not parameter.sliced and is_prefill_phase and is_opt_shard_group:
+            rank = get_rank()
+            new_tensor = _load_tensor_by_layout(parameter, layout, rank)
+            parameter.set_data(new_tensor, True)
+            return
+        layout_shape = layout[2]
+        parameter.shape = tuple(layout_shape)
+        return
     graph_executor = GraphExecutor_.get_instance()
     new_param = parameter.init_data(layout, set_sliced=True)
     parameter = new_param
@@ -135,7 +149,7 @@ def _to_full_shapes(shapes, device_num):
             if len(shape) != len(dataset_strategy[index]):
                 raise ValueError("The input shapes item size {} is not equal to "
                                  "dataset strategy item size {}".format(len(shape), len(dataset_strategy[index])))
-            new_shape = ()
+            new_shape = []
             for i, item in enumerate(shape):
                 if item > 0:
                     new_shape += (item * dataset_strategy[index][i],)  # static shape
@@ -144,16 +158,69 @@ def _to_full_shapes(shapes, device_num):
             new_shapes.append(new_shape)
         return new_shapes
     for shape in shapes:
-        new_shape = ()
+        shape_v = []
+        for i, item in enumerate(shape):
+            if i == 0 and item > 0:
+                shape_v += (item * device_num,)  # only for static shape
+            else:
+                shape_v += (item,)
+        new_shapes.append(shape_v)
+    return new_shapes
+
+
+def _origin_shapes(shapes):
+    """resume origin shape after full shape."""
+    if _need_to_full():
+        device_num = _get_device_num() // _get_pipeline_stages()
+    else:
+        return shapes
+    new_shapes = []
+    dataset_strategy = ()
+    if context.get_auto_parallel_context("dataset_strategy") not in ("data_parallel", "full_batch"):
+        dataset_strategy = context.get_auto_parallel_context("dataset_strategy")
+    if dataset_strategy:
+        if len(shapes) != len(dataset_strategy):
+            raise ValueError("The input shapes size {} is not equal to "
+                             "dataset strategy size {}".format(len(shapes), len(dataset_strategy)))
+        for index, shape in enumerate(shapes):
+            if len(shape) != len(dataset_strategy[index]):
+                raise ValueError("The input shapes item size {} is not equal to "
+                                 "dataset strategy item size {}".format(len(shape), len(dataset_strategy[index])))
+            new_shape = []
+            for i, item in enumerate(shape):
+                if item > 0:
+                    new_shape += (item // dataset_strategy[index][i],)  # static shape
+                else:
+                    new_shape += (item,)  # dynamic shape
+            new_shapes.append(new_shape)
+        return new_shapes
+    for shape in shapes:
+        shape_v = []
         for i, item in enumerate(shape):
             if i == 0 and item > 0:
-                new_shape += (item * device_num,)  # only for static shape
+                shape_v += (item // device_num,)  # only for static shape
             else:
-                new_shape += (item,)
-        new_shapes.append(new_shape)
+                shape_v += (item,)
+        new_shapes.append(shape_v)
     return new_shapes
 
 
+def _dynamic_shape_for_dataset(dataset_shapes, dynamic_shapes):
+    """convert static dataset shapes to dynamic shape"""
+    if len(dataset_shapes) != len(dynamic_shapes):
+        raise ValueError("The dataset shapes size of {} is not equal to "
+                         "dynamic shapes size of {}".format(dataset_shapes, dynamic_shapes))
+    ret = dataset_shapes
+    for i in range(len(dynamic_shapes)):
+        if len(dataset_shapes[i]) != len(dynamic_shapes[i]):
+            raise ValueError("The dataset shapes size of {} is not equal to "
+                             "dynamic shapes size of {}".format(dataset_shapes, dynamic_shapes))
+        for j in range(len(dynamic_shapes[i])):
+            if dynamic_shapes[i][j] == -1:
+                ret[i][j] = -1
+    return ret
+
+
 def _to_full_tensor(elem, global_device_num, global_rank, scaling_sens=None):
     """Convert numpy to tensor, expanding batch dimension according to device_num, adapt to feed the data
        from host solution.
@@ -418,6 +485,94 @@ def _infer_rank_list(train_map, predict_map=None):
     return ret
 
 
+def _handle_symbol_inputs(symbol_inputs):
+    """handle symbol inputs"""
+    dataset_strategy = ()
+    divisor_key = "divisor"
+    # dataset strategy is set
+    if context.get_auto_parallel_context("dataset_strategy") not in ("data_parallel", "full_batch"):
+        dataset_strategy = context.get_auto_parallel_context("dataset_strategy")
+    if dataset_strategy:
+        if len(symbol_inputs) != len(dataset_strategy):
+            raise ValueError("The symbol_inputs size {} is not equal to "
+                             "dataset strategy size {}".format(len(symbol_inputs), len(dataset_strategy)))
+        for index, shape in enumerate(symbol_inputs):
+            dataset_ele_s = dataset_strategy[index]
+            if len(shape) != len(dataset_ele_s):
+                raise ValueError("The symbol_inputs item size {} is not equal to "
+                                 "dataset strategy item size {}".format(len(shape), len(dataset_ele_s)))
+
+            for i, item in enumerate(shape):
+                if isinstance(item, dict):  # symbol
+                    symbol_inputs[index][i][divisor_key] = symbol_inputs[index][i][divisor_key] * dataset_ele_s[i]
+                else:  # common shape
+                    symbol_inputs[index][i] = item * dataset_ele_s[i]
+
+        return symbol_inputs
+
+    # full batch is set
+    device_num = _get_device_num() // _get_pipeline_stages()
+    for index, shape in enumerate(symbol_inputs):
+        for i, item in enumerate(shape):
+            if i == 0 and isinstance(item, dict):  # symbol
+                symbol_inputs[index][i][divisor_key] = symbol_inputs[index][i][divisor_key] * device_num
+
+    return symbol_inputs
+
+
+def _no_need_to_change_symbols(shapes):
+    """no need to handle the symbol if full_batch is true or it's not parallel mode"""
+    if not _need_to_full():
+        return True
+
+    # if static shape, return
+    is_dynamic_shape = False
+    for shape in shapes:
+        if any(i < 0 for i in shape):
+            is_dynamic_shape = True
+            break
+    if is_dynamic_shape is False:
+        return True
+
+    return False
+
+
+def _change_symbols_for_parallel(shapes, symbol_inputs=None):
+    """create or modify symbol inputs"""
+    if _no_need_to_change_symbols(shapes) is True:
+        return symbol_inputs
+    # the symbol_inputs is [[{'divisor': 8}, 16], [{'divisor': 8}, 16]]
+    # the dataset_shapes is [(-1, 16), (-1, 16)]
+    # if symbol_inputs is [None, None, ..., None], reset it
+    if symbol_inputs is not None and all(s is None for s in symbol_inputs):
+        symbol_inputs = []
+
+    # if symbol inputs is none or empty, create default symbol inputs
+    # if symbol inputs is not none, handle the empty symbol
+    divisor_key = "divisor"
+    if symbol_inputs is None or bool(symbol_inputs) is False:
+        symbol_inputs = [list(shape) for shape in shapes]  # tuple to list
+        for i, s in enumerate(symbol_inputs):
+            for j, item in enumerate(s):
+                if item == -1:
+                    symbol_inputs[i][j] = {divisor_key: 1}
+    else:
+        for i, s in enumerate(symbol_inputs):
+            # the symbol_inputs may be [None, [{'divisor': 8}, 16]]
+            # and the dataset_shapes is [(-1, 16), (-1, 16)], need to handle None
+            if s is None:
+                symbol_inputs[i] = shapes[i]
+                for k, item in enumerate(symbol_inputs[i]):
+                    if item == -1:
+                        symbol_inputs[i][k] = {divisor_key: 1}
+                s = symbol_inputs[i]
+            for j, item in enumerate(s):
+                if isinstance(item, dict) and bool(item) is False:  # the item is empty
+                    symbol_inputs[i][j] = {divisor_key: 1}
+
+    return _handle_symbol_inputs(symbol_inputs)
+
+
 def _grads_divided_by_device_num_if_recomputation(grads):
     """
     If in pynative parallel and full_batch is True, divide grads by device num to ensure that the gradients is correct.

mindspore/parallel/algo_parameter_config.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Huawei Technologies Co., Ltd
+# Copyright 2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -227,9 +227,7 @@ get_algo_parameters_config_func_map = {
                  enable_algo_approxi=bool, algo_approxi_epsilon=float)
 def set_algo_parameters(**kwargs):
     """
-    Set parameters in the algorithm for parallel strategy searching. See a typical use in
-    `test_auto_parallel_resnet.py
-    <https://gitee.com/mindspore/mindspore/blob/r2.2/tests/ut/python/parallel/test_auto_parallel_resnet.py>`_.
+    Set parameters in the algorithm for parallel strategy searching.
 
     Note:
         The attribute name is required. This interface works ONLY in AUTO_PARALLEL mode.
@@ -266,14 +264,14 @@ def set_algo_parameters(**kwargs):
 
             For the Ascend devices, users need to prepare the rank table, set rank_id and device_id.
             Please see the `rank table startup
-            <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/rank_table.html>`_
+            <https://www.mindspore.cn/tutorials/experts/en/master/parallel/rank_table.html>`_
             for more details.
 
             For the GPU devices, users need to prepare the host file and mpi, please see the `mpirun startup
-            <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/mpirun.html>`_ .
+            <https://www.mindspore.cn/tutorials/experts/en/master/parallel/mpirun.html>`_ .
 
             For the CPU device, users need to write a dynamic cluster startup script, please see the `Dynamic Cluster
-            Startup <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/dynamic_cluster.html>`_ .
+            Startup <https://www.mindspore.cn/tutorials/experts/en/master/parallel/dynamic_cluster.html>`_ .
 
         >>> import numpy as np
         >>> import mindspore as ms
@@ -284,7 +282,7 @@ def set_algo_parameters(**kwargs):
         >>>
         >>> ms.set_context(mode=ms.GRAPH_MODE)
         >>> ms.set_auto_parallel_context(parallel_mode=ms.ParallelMode.AUTO_PARALLEL,
-        >>>                              search_mode="sharding_propagation")
+        ...                              search_mode="sharding_propagation")
         >>> init()
         >>> ms.set_algo_parameters(fully_use_devices=True)
         >>> ms.set_algo_parameters(elementwise_op_strategy_follow=True)

mindspore/parallel/checkpoint_transform.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Huawei Technologies Co., Ltd
+# Copyright 2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -22,6 +22,7 @@ from collections import defaultdict
 import numpy as np
 import mindspore as ms
 from mindspore.common import dtype as mstype
+from mindspore.parallel._utils import _is_in_auto_parallel_mode
 from mindspore.parallel._parallel_serialization import _rank_list_for_transform_parallel_checkpoint, \
     _transform_parallel_checkpoint, _get_device_num_from_strategy, _make_dir, \
     _extract_layout_map, _extract_src_dst_layout_map, _parameter_not_in_local_stage, _extract_pipeline_stage_num, \
@@ -29,28 +30,28 @@ from mindspore.parallel._parallel_serialization import _rank_list_for_transform_
 
 
 __all__ = ["merge_pipeline_strategys", "rank_list_for_transform", "transform_checkpoint_by_rank",
-           "transform_checkpoints", "load_segmented_checkpoints"]
+           "transform_checkpoints", "sync_pipeline_shared_parameters", "load_segmented_checkpoints"]
 
 
 def merge_pipeline_strategys(src_strategy_dirs, dst_strategy_file):
     """
     Merge parallel strategy between all pipeline stages in pipeline parallel mode.
     For more details about converting distributed Checkpoint, please refer to
-    `Model Transformation <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/model_transformation.html>`_.
+    `Model Transformation <https://www.mindspore.cn/tutorials/experts/en/master/parallel/model_transformation.html>`_.
 
     Note:
         Strategy file of each pipeline stage should be included in src_strategy_dirs.
 
     Args:
         src_strategy_dirs (str): The directory of strategy files including all pipeline stage which is saved by
-                                 'mindspore.set_auto_parallel_context(strategy_ckpt_save_file)'
+                                 'mindspore.set_auto_parallel_context(strategy_ckpt_save_file)'.
         dst_strategy_file (str): The file merged strategy to save.
 
     Raises:
         NotADirectoryError: `src_strategy_dirs` is not a directory.
 
     Examples:
-        >>> import mindspore ms
+        >>> import mindspore as ms
         >>> # src_strategy_dir/stra0.ckpt, src_strategy_dir/stra1.ckpt ... src_strategy_dir/stra127.ckpt
         >>> ms.merge_pipeline_strategys("./src_strategy_dir", "./dst_strategy.ckpt")
 
@@ -71,30 +72,32 @@ def merge_pipeline_strategys(src_strategy_dirs, dst_strategy_file):
         _merge_json_strategy(src_strategy_files_json, dst_strategy_file)
 
 
+
 def rank_list_for_transform(rank_id, src_strategy_file=None, dst_strategy_file=None):
     """
-    List of original distributed checkpoint rank index for obtaining the target checkpoint of a rank_id
-    during the distributed checkpoint conversion. For more details about converting distributed Checkpoint,
-    please refer to `Model Transformation <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/model_transformation.html>`_.
+    List of original distributed checkpoint rank index for obtaining the target checkpoint of a rank_id during the
+    distributed checkpoint conversion. For more details about converting distributed Checkpoint, please refer to
+    `Model Transformation <https://www.mindspore.cn/tutorials/experts/en/master/parallel/model_transformation.html>`_.
 
     Args:
         rank_id (int): The rank of which distributed checkpoint needs to be obtained after conversion.
         src_strategy_file (str): Name of source sharding strategy file which saved by
                                  `mindspore.set_auto_parallel_context(strategy_ckpt_save_file)`.
-                                 when the 'src_strategy_file' is None, it means that the source sharding strategy is
-                                 without any sharing for each parameter. Default:None.
+                                 when the `src_strategy_file` is ``None``, it means that the source sharding strategy is
+                                 without any sharing for each parameter. Default: ``None``.
         dst_strategy_file (str): Name of destination sharding strategy file which saved by
                                  `mindspore.set_auto_parallel_context(strategy_ckpt_save_file)`.
-                                 when the 'dst_strategy_file' is None, it means that the destination sharding strategy
-                                 is without any sharing for each parameter. Default:None.
+                                 when the `dst_strategy_file` is ``None``,
+                                 it means that the destination sharding strategy
+                                 is without any sharing for each parameter. Default: ``None``.
 
     Returns:
         List, the rank list required for converting the distributed checkpoint of rank_id.
 
     Raises:
-        ValueError: `src_strategy_file` or dst_strategy_file is incorrect.
-        TypeError: `src_strategy_file` or dst_strategy_file is not a string.
-        TypeError: `rank_id` is not a int.
+        ValueError: `src_strategy_file` or `dst_strategy_file` is incorrect.
+        TypeError: `src_strategy_file` or `dst_strategy_file` is not a string.
+        TypeError: `rank_id` is not an int.
 
     Examples:
         >>> import mindspore as ms
@@ -137,7 +140,7 @@ def transform_checkpoint_by_rank(rank_id, checkpoint_files_map, save_checkpoint_
     """
     Transform distributed checkpoint from source sharding strategy to destination sharding strategy by rank
     for a network. For more details about converting distributed Checkpoint, please refer to
-    `Model Transformation <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/model_transformation.html>`_.
+    `Model Transformation <https://www.mindspore.cn/tutorials/experts/en/master/parallel/model_transformation.html>`_.
 
     Args:
         rank_id (int): The rank of which distributed checkpoint needs to be obtained after conversion.
@@ -146,12 +149,13 @@ def transform_checkpoint_by_rank(rank_id, checkpoint_files_map, save_checkpoint_
         save_checkpoint_file_name (str): The file name to save the converted checkpoint.
         src_strategy_file (str): Name of source sharding strategy file which saved by
                                  'mindspore.set_auto_parallel_context(strategy_ckpt_save_file)'.
-                                 when the 'src_strategy_file' is None, it means that the source sharding strategy is
-                                 without any sharing for each parameter. Default:None.
+                                 when the `src_strategy_file` is None, it means that the source sharding strategy is
+                                 without any sharing for each parameter. Default: ``None``.
         dst_strategy_file (str): Name of destination sharding strategy file which saved by
                                  'mindspore.set_auto_parallel_context(strategy_ckpt_save_file)'.
-                                 when the 'dst_strategy_file' is None, it means that the destination sharding strategy
-                                 is without any sharing for each parameter. Default:None.
+                                 when the `dst_strategy_file` is ``None``,
+                                 it means that the destination sharding strategy
+                                 is without any sharing for each parameter. Default: ``None``.
 
     Raises:
         ValueError: `src_strategy_file` or `dst_strategy_file` is incorrect.
@@ -159,7 +163,7 @@ def transform_checkpoint_by_rank(rank_id, checkpoint_files_map, save_checkpoint_
         ValueError: `save_checkpoint_file_name` is not end with ".ckpt".
         TypeError: `checkpoint_files_map` is not a dict.
         TypeError: `src_strategy_file` or `dst_strategy_file` is not a string.
-        TypeError: `rank_id` is not a int.
+        TypeError: `rank_id` is not an int.
         TypeError: `save_checkpoint_file_name` is not a string.
 
     Examples:
@@ -357,7 +361,7 @@ def transform_checkpoints(src_checkpoints_dir, dst_checkpoints_dir, ckpt_prefix,
     """
     Transform distributed checkpoint from source sharding strategy to destination sharding strategy for a rank.
     For more details about converting distributed Checkpoint, please refer to
-    `Model Transformation <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/model_transformation.html>`_.
+    `Model Transformation <https://www.mindspore.cn/tutorials/experts/en/master/parallel/model_transformation.html>`_.
 
     Note:
         The `src_checkpoints_dir` directory structure should be organized like "src_checkpoints_dir/rank_0/a.ckpt", the
@@ -405,23 +409,178 @@ def transform_checkpoints(src_checkpoints_dir, dst_checkpoints_dir, ckpt_prefix,
     src_layout_map = _extract_layout_map(src_strategy_file)
     dst_layout_map = _extract_layout_map(dst_strategy_file)
     pipeline_stage_num = _extract_pipeline_stage_num(src_strategy_file)
+    dst_stage_num = _extract_pipeline_stage_num(dst_strategy_file)
     if src_layout_map:
-        src_param_keys = {param_name for param_name in src_layout_map if not param_name.startswith("accu_grads")}
+        src_param_keys = {param_name for param_name in src_layout_map if
+                          not param_name.startswith(("accu_grads", "adam_v", "adam_m"))}
     if dst_layout_map:
-        dst_param_keys = {param_name for param_name in dst_layout_map if not param_name.startswith("accu_grads")}
-    if src_layout_map and dst_layout_map and pipeline_stage_num == 1 \
-        and src_param_keys.issubset(dst_param_keys) and len(src_param_keys) < len(dst_param_keys):
-        dst_stage_num = _extract_pipeline_stage_num(dst_strategy_file)
-        if dst_stage_num > 1:
-            raise NotImplementedError("When using unmerged src strategy, dst strategy doesn't \
-                                       support strategy with pipeline parallel.")
+        dst_param_keys = {param_name for param_name in dst_layout_map if
+                          not param_name.startswith(("accu_grads", "adam_v", "adam_m"))}
+    layout_is_passed = src_layout_map and dst_layout_map
+
+    if layout_is_passed and pipeline_stage_num == 1 and dst_stage_num == 1 and \
+        src_param_keys.issubset(dst_param_keys) and len(src_param_keys) < len(dst_param_keys):
+        ms.log.info("Transform checkpoint by every pipeline stage.")
         _transform_checkpoint_by_stage(src_checkpoints_dir, dst_checkpoints_dir, ckpt_prefix,
                                        src_strategy_file, dst_strategy_file)
     else:
+        ms.log.info("Transform checkpoints by all pipeline stage.")
         _transform_checkpoints(src_checkpoints_dir, dst_checkpoints_dir, ckpt_prefix,
                                src_strategy_file, dst_strategy_file)
 
 
+def _sync_params(name, param, layout):
+    """synchronize single parameter"""
+    if len(layout) < 10:
+        ms.log.warning("The layout dict does not contain the pipeline_shared_param info %s", name)
+        return
+
+    pipeline_shared = layout[8]
+    if not pipeline_shared:
+        return
+
+    is_send = layout[9]
+    peer_rank = layout[10]
+    sr_tag = layout[11]
+
+    class SharedParameterSyncCell(ms.nn.Cell):
+        """synchronize cell"""
+        def __init__(self, param, is_send, peer_rank, sr_tag):
+            super().__init__()
+            self.param = param
+            self.is_send = is_send
+            self.ret = ms.Tensor([0])
+
+            from mindspore.ops import Send, Receive
+            if self.is_send:
+                self.send = Send(sr_tag=sr_tag, dest_rank=peer_rank)
+            else:
+                self.receive = Receive(sr_tag=sr_tag, src_rank=peer_rank, shape=param.shape, dtype=param.dtype)
+
+        def construct(self):
+            if self.is_send:
+                out = self.send(self.param)
+                return ms.ops.functional.depend(self.ret, out)
+
+            self.param = self.receive(self.ret)
+            return ms.ops.functional.depend(self.ret, self.param)
+
+    sync_net = SharedParameterSyncCell(param, is_send, peer_rank, sr_tag)
+    sync_net()
+
+
+def sync_pipeline_shared_parameters(net):
+    """synchronize pipeline parallel stage shared parameters.
+    Parameters may be shared between different stages. For example, `embedding table` is
+    shared by `WordEmbedding` layer and `LMHead` layer, which are usually split into different stages. It is necessary
+    to perform synchronization after `embedding table` changes.
+
+    Note:
+        The network should be compiled before synchronize pipeline parallel stage shared parameters.
+
+    Args:
+        net (nn.Cell): the inference network.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        .. note::
+            Before running the following examples, you need to configure the communication environment variables.
+
+            For the Ascend device, users need to write a dynamic cluster startup script, please see the `Dynamic Cluster
+            Startup <https://www.mindspore.cn/tutorials/experts/en/master/parallel/dynamic_cluster.html>`_ .
+
+        >>> import numpy as np
+        >>> import mindspore as ms
+        >>> import mindspore.communication.management as D
+        >>> from mindspore import lazy_inline, context, nn, ops, Parameter, Tensor
+        >>> context.set_context(mode=context.GRAPH_MODE)
+        >>> class Embedding(nn.Cell):
+        ...     def __init__(self, shape):
+        ...         super().__init__()
+        ...         self.w = Parameter(Tensor(np.ones(shape), ms.float32), name='w')
+        ...         self.matmul = ops.MatMul().shard(((1, 1), (1, 1)))
+        ...     def construct(self, x):
+        ...         return self.matmul(x, self.w), self.w
+        ...
+        >>> class LMHead(nn.Cell):
+        ...     def __init__(self):
+        ...         super().__init__()
+        ...         self.matmul = ops.MatMul(transpose_b=True).shard(((1, 1), (1, 1)))
+        ...     def construct(self, x, w):
+        ...         return self.matmul(x, w)
+        ...
+        >>> class Network(nn.Cell):
+        ...     @lazy_inline
+        ...     def __init__(self):
+        ...         super().__init__()
+        ...         shape = (4, 4)
+        ...         self.word_embedding = Embedding(shape)
+        ...         self.lm_head = LMHead()
+        ...         self.word_embedding.pipeline_stage = 0
+        ...         self.lm_head.pipeline_stage = 1
+        ...     def construct(self, x):
+        ...         x, embed = self.word_embedding(x)
+        ...         return self.lm_head(x, embed)
+        ...
+        >>> class PipelineCellInference(nn.Cell):
+        ...     def __init__(self, network, micro_batch_num):
+        ...         super().__init__()
+        ...         self.network = network
+        ...         self.micro_batch_num = micro_batch_num
+        ...         self.concat = ops.Concat()
+        ...     def construct(self, x):
+        ...         ret = ()
+        ...         for i in range(self.micro_batch_num):
+        ...             micro_batch_size = x.shape[0] // self.micro_batch_num
+        ...             start = micro_batch_size * i
+        ...             end = micro_batch_size * (i + 1)
+        ...             micro_input = x[start:end]
+        ...             y = self.network(micro_input)
+        ...             ret = ret + (y,)
+        ...         ret = self.concat(ret)
+        ...         return ret
+        >>> D.init()
+        >>> context.set_auto_parallel_context(parallel_mode='semi_auto_parallel', full_batch=True, pipeline_stages=2)
+        >>> net = Network()
+        >>> net = PipelineCellInference(net, 2)
+        >>> net.set_train(False)
+        >>> x = Tensor(np.ones((2, 4)), ms.float32)
+        >>> net.compile(x)
+        >>> ms.sync_pipeline_shared_parameters(net)
+        >>> print(net.network.word_embedding.w.asnumpy())
+        [[1. 1. 1. 1.]
+         [1. 1. 1. 1.]
+         [1. 1. 1. 1.]
+         [1. 1. 1. 1.]]
+    """
+
+    if not isinstance(net, ms.nn.Cell):
+        ms.log.critical("Failed to synchronize pipeline shared parameters.")
+        msg = ("For 'sync_pipeline_shared_parameters', the argument 'net' should be a Cell, "
+               "but got {}.".format(type(net)))
+        raise TypeError(msg)
+
+    layout_dict = net.parameter_layout_dict
+    if _is_in_auto_parallel_mode() and not layout_dict:
+        from mindspore.common.api import _get_parameter_layout
+        layout_dict = _get_parameter_layout()
+
+    # switch to standalone mode
+    parallel_mode = ms.context.get_auto_parallel_context("parallel_mode")
+    full_batch = ms.context.get_auto_parallel_context("full_batch")
+    ms.context.set_auto_parallel_context(parallel_mode="stand_alone", full_batch=False)
+
+    # synchronize shared parameter
+    for name, param in net.parameters_and_names():
+        if name in layout_dict:
+            _sync_params(name, param, layout_dict[name])
+
+    # restore parallel context
+    ms.context.set_auto_parallel_context(parallel_mode=parallel_mode, full_batch=full_batch)
+
+
 def load_segmented_checkpoints(ckpt_file_dir, net=None, strict_load=False, filter_prefix=None,
                                dec_key=None, dec_mode="AES-GCM", specify_prefix=None, choice_func=None):
     """
@@ -473,8 +632,8 @@ def load_segmented_checkpoints(ckpt_file_dir, net=None, strict_load=False, filte
     if not isinstance(ckpt_file_dir, str):
         raise TypeError("The ckpt_file_dir should be a str.")
     if not os.path.isdir(ckpt_file_dir):
-        raise ValueError("The dst_strategy_file: {} doesn't exists. Or it's not a directory.".
-                         format(os.path.dirname(ckpt_file_dir)))
+        raise ValueError("The dst_strategy_file: {} doesn't exist. Or it's not a directory".
+                         format(ckpt_file_dir))
     checkpoint_file_name = os.path.join(ckpt_file_dir, "*.ckpt")
     rank_ckpts = glob.glob(checkpoint_file_name)
     parameter_dict = {}

mindspore/parallel/cluster/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Entry for ms_run"""

mindspore/parallel/cluster/process_entity/__init__.py

@@ -0,0 +1,18 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Interfaces for ms_run"""
+from ._api import _Node, _MetaServerNode, _ComputeGraphNode, _ProcessManager
+
+from ._utils import _generate_cmd, _generate_url, _is_local_ip, _send_scale_num, _get_status_and_params