mindspore 2.0.0a0__cp38-cp38-win_amd64.whl → 2.0.0rc1__cp38-cp38-win_amd64.whl

mindspore/{nn/transformer → parallel/_transformer}/transformer.py

@@ -30,18 +30,17 @@ import mindspore.common.dtype as mstype
 from mindspore.ops import operations as P
 from mindspore.ops import functional as F
 from mindspore.nn.cell import Cell
-from mindspore._checkparam import Validator
+from mindspore import _checkparam as Validator
 from mindspore import log as logger
-from mindspore.parallel._utils import _get_parallel_mode, _is_sharding_propagation
+from mindspore.parallel._utils import _get_parallel_mode
 from mindspore.context import ParallelMode
 from mindspore.log import _LogActionOnce
-from mindspore.nn.transformer.layers import _LayerNorm, _Linear, _check_input_shape, \
+from mindspore.parallel._transformer.layers import _LayerNorm, _Linear, \
     _args_type_validator_check, _valid_type_checks, _valid_value_checks, \
-    _check_shape_equal, _check_past_none_input_none, _check_input_dtype, _check_input_shape_value, \
-    _check_shape_equal_without_batch
-from mindspore.nn.transformer.op_parallel_config import default_dpmp_config, _PipeLineConfig, OpParallelConfig, \
+    _check_past_none_input_none, _check_input_dtype
+from mindspore.parallel._transformer.op_parallel_config import default_dpmp_config, _PipeLineConfig, OpParallelConfig, \
     _Config, _check_config, MoEParallelConfig
-from mindspore.nn.transformer.moe import default_moe_config, MoE, _check_moe_config
+from mindspore.parallel._transformer.moe import default_moe_config, MoE, _check_moe_config
 
 __all__ = [
     "AttentionMask",
@@ -399,13 +398,13 @@ class FeedForward(Cell):
             (2, 20, 15)
             >>> # Example 2 using custom hidden activation
             >>> class MyActivationNoShard(nn.Cell):
-            >>>     def __init__(self):
-            >>>         super(MyActivationNoShard, self).__init__()
-            >>>         self.add = ops.Add()
-            >>>     def construct(self, x):
-            >>>         return self.add(x, 0.1)
+            ...     def __init__(self):
+            ...         super(MyActivationNoShard, self).__init__()
+            ...         self.add = ops.Add()
+            ...     def construct(self, x):
+            ...         return self.add(x, 0.1)
             >>> model = FeedForward(hidden_size=15, ffn_hidden_size=30, dropout_rate=0.1,
-            >>>                     hidden_act=MyActivationNoShard)
+            ...                     hidden_act=MyActivationNoShard)
             >>> tensor = Tensor(np.ones((2, 20, 15)), mstype.float32)
             >>> output = model(tensor)
             >>> print(output.shape)
@@ -415,16 +414,16 @@ class FeedForward(Cell):
             >>> # a class function named activation_shard. It accepts the argument parallel_config (OpParallelConfig,
             >>> # MoEParallelConfig) and set the shard for the primitives used in the construct.
             >>> class MyActivationWithShard(nn.Cell):
-            >>>     def __init__(self):
-            >>>         super(MyActivationWithShard, self).__init__()
-            >>>         self.add = ops.Add()
-            >>>     def construct(self, x):
-            >>>         return self.add(x, 0.1)
-            >>>     def activation_shard(self, parallel_config):
-            >>>         self.add.shard(((parallel_config.data_parallel, parallel_config.model_parallel), ()))
+            ...     def __init__(self):
+            ...         super(MyActivationWithShard, self).__init__()
+            ...         self.add = ops.Add()
+            ...     def construct(self, x):
+            ...         return self.add(x, 0.1)
+            ...     def activation_shard(self, parallel_config):
+            ...         self.add.shard(((parallel_config.data_parallel, parallel_config.model_parallel), ()))
             >>>
             >>> model = FeedForward(hidden_size=15, ffn_hidden_size=30, dropout_rate=0.1,
-            >>>                     hidden_act=MyActivationWithShard)
+            ...                     hidden_act=MyActivationWithShard)
             >>> tensor = Tensor(np.ones((2, 20, 15)), mstype.float32)
             >>> output = model(tensor)
             >>> print(output.shape)
@@ -451,7 +450,7 @@ class FeedForward(Cell):
         if hidden_act is None or not (isinstance(hidden_act, str) or issubclass(hidden_act, nn.Cell)):
             raise TypeError(f"For FeedForward cell, the hidden_act should str type or nn.Cell type, "
                             f"but got {hidden_act}.")
-        if _get_parallel_mode() in (ParallelMode.AUTO_PARALLEL,) and _is_sharding_propagation():
+        if _get_parallel_mode() in (ParallelMode.AUTO_PARALLEL,):
             _check_config(parallel_config)
             mp = parallel_config.model_parallel
             if expert_num > 1:
@@ -497,9 +496,9 @@ class FeedForward(Cell):
             else:
                 self.projection.shard(strategy_matmul=((dp, mp), (mp, 1)))
             self.projection.bias.parallel_optimizer = False
-            self.dropout = nn.Dropout(1 - dropout_rate)
-            self.dropout_3d = nn.Dropout(1 - dropout_rate)
-            self.dropout_4d = nn.Dropout(1 - dropout_rate)
+            self.dropout = nn.Dropout(p=dropout_rate)
+            self.dropout_3d = nn.Dropout(p=dropout_rate)
+            self.dropout_4d = nn.Dropout(p=dropout_rate)
             self.cast = P.Cast()
         else:
             _check_config(parallel_config)
@@ -557,16 +556,18 @@ class FeedForward(Cell):
                 self.projection.shard(strategy_matmul=((dp, mp), (mp, 1)),
                                       strategy_bias=((dp, 1), (1,)))
             self.projection.bias.parallel_optimizer = False
-            self.dropout = nn.Dropout(1 - dropout_rate)
+            self.dropout = nn.Dropout(p=dropout_rate)
             self.dropout.dropout.shard(((dp, 1),))
-            self.dropout_3d = nn.Dropout(1 - dropout_rate)
+            self.dropout_3d = nn.Dropout(p=dropout_rate)
             self.dropout_3d.dropout.shard(((dp, 1, 1),))
-            self.dropout_4d = nn.Dropout(1 - dropout_rate)
+            self.dropout_4d = nn.Dropout(p=dropout_rate)
             self.dropout_4d.dropout.shard(((dp, ep, 1, 1),))
             self.cast = P.Cast()
+            # for grouped pairwise exchange alltoall method in pass
+            self.mapping.matmul.add_prim_attr("gpea_label", True)
+            self.projection.matmul.add_prim_attr("gpea_label", True)
 
     def construct(self, x):
-        _check_input_shape(F.shape(x), "x", self.cls_name, [2, 3])
         _check_input_dtype(F.dtype(x), "x", [mstype.float32, mstype.float16], self.cls_name)
         x = self.cast(x, mstype.float16)
         # returned shape is [bs, seq_length, ffn_hidden_size] or [bs * seq_length, ffn_hidden_size]
@@ -639,9 +640,7 @@ class AttentionMask(Cell):
         self.multiply = P.Mul().shard(((parallel_config.data_parallel, 1, 1), (1, 1, 1)))
 
     def construct(self, input_mask):
-        _check_input_shape(F.shape(input_mask), "input_mask", self.cls_name, 2)
         _check_input_dtype(F.dtype(input_mask), "input_mask", [mstype.float32, mstype.float16], self.cls_name)
-        _check_input_shape_value(F.shape(input_mask), 1, "input_mask", self.cls_name, self.seq_length)
         input_mask = P.Cast()(self.not_equal(input_mask, 0), mstype.float16)
         input_shape = P.Shape()(input_mask)
         shape_right = (input_shape[0], 1, input_shape[1])
@@ -736,7 +735,6 @@ class VocabEmbedding(Cell):
                         f"model parallel for the embedding lookup.")
 
     def construct(self, input_ids):
-        _check_input_shape(F.shape(input_ids), "input_ids", self.cls_name, 2)
         _check_input_dtype(F.dtype(input_ids), "input_ids", [mstype.int32], self.cls_name)
         output = self.gather(self.embedding_table, input_ids, 0)
         return output, self.embedding_table.value()
@@ -904,7 +902,7 @@ class MultiHeadAttention(Cell):
             ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL)
         if batch_size:
             Validator.check_positive_int(batch_size)
-        if _get_parallel_mode() in (ParallelMode.AUTO_PARALLEL,) and _is_sharding_propagation():
+        if _get_parallel_mode() in (ParallelMode.AUTO_PARALLEL,):
             _check_config(parallel_config)
             self.src_seq_length = src_seq_length
             self.tgt_seq_length = tgt_seq_length
@@ -955,8 +953,8 @@ class MultiHeadAttention(Cell):
             # Normalize factor for attention, sqrt(dk) as widely used
             self.scale_factor = Tensor(math.sqrt(math.sqrt(self.size_per_head)))
             self.use_past = use_past
-            self.dropout = nn.Dropout(1 - hidden_dropout_rate)
-            self.prob_dropout = nn.Dropout(1 - attention_dropout_rate)
+            self.dropout = nn.Dropout(p=hidden_dropout_rate)
+            self.prob_dropout = nn.Dropout(p=attention_dropout_rate)
             self.softmax = nn.Softmax().to_float(softmax_compute_type)
             self.softmax_3d = nn.Softmax().to_float(softmax_compute_type)
             self.expand_dims = P.ExpandDims()
@@ -1056,9 +1054,9 @@ class MultiHeadAttention(Cell):
             # Normalize factor for attention, sqrt(dk) as widely used
             self.scale_factor = Tensor(math.sqrt(math.sqrt(self.size_per_head)))
             self.use_past = use_past
-            self.dropout = nn.Dropout(1 - hidden_dropout_rate)
+            self.dropout = nn.Dropout(p=hidden_dropout_rate)
             self.dropout.dropout.shard(((parallel_config.data_parallel, 1),))
-            self.prob_dropout = nn.Dropout(1 - attention_dropout_rate)
+            self.prob_dropout = nn.Dropout(p=attention_dropout_rate)
             self.prob_dropout.dropout.shard(
                 ((parallel_config.data_parallel, parallel_config.model_parallel, 1, 1),))
             self.softmax = nn.Softmax().to_float(softmax_compute_type)
@@ -1204,46 +1202,22 @@ class MultiHeadAttention(Cell):
 
     def _get_batch_size_from_query(self, query):
         r"""Get the batch size from query tensor"""
-        batch_size = None
         # For the incremental prediction, the seq length for the input is 1.
-        if len(F.shape(query)) == 2 and self.is_first_iteration:
-            batch_size = F.shape(query)[0] // self.src_seq_length
-        else:
-            batch_size = F.shape(query)[0]
-        return batch_size
+        if len(F.shape(query)) == 2 and ((self.use_past and self.is_first_iteration) or (not self.use_past)):
+            return F.shape(query)[0] // self.src_seq_length
+        return F.shape(query)[0]
 
     def _get_seq_length_under_incremental(self, length):
         r"""Return the length of the tensor.
             For the incremental prediction, the seq length for the input is 1.
         """
-        if self.is_first_iteration:
-            return length
-        return 1
+        if self.use_past and not self.is_first_iteration:
+            return 1
+        return length
 
     def _check_inputs(self, query_tensor, key_tensor, value_tensor, attention_mask, key_past=None,
                       value_past=None, batch_valid_length=None):
         r"""Check inputs"""
-        if not self.use_past or (self.use_past and self.is_first_iteration):
-            _check_shape_equal_without_batch(F.shape(query_tensor), "query_tensor", self.cls_name,
-                                             [self.src_seq_length, self.hidden_size])
-            _check_shape_equal_without_batch(F.shape(key_tensor), "key_tensor", self.cls_name,
-                                             [self.tgt_seq_length, self.hidden_size])
-            _check_shape_equal_without_batch(F.shape(value_tensor), "value_tensor", self.cls_name,
-                                             [self.tgt_seq_length, self.hidden_size])
-            if attention_mask is not None:
-                _check_shape_equal(F.shape(attention_mask), "attention_mask", self.cls_name,
-                                   [F.shape(attention_mask)[0], self.src_seq_length, self.tgt_seq_length])
-        else:
-            _check_shape_equal(F.shape(query_tensor), "query_tensor", self.cls_name,
-                               [[self.batch_size, 1, self.hidden_size], [self.batch_size, self.hidden_size]])
-            _check_shape_equal(F.shape(key_tensor), "key_tensor", self.cls_name,
-                               [[self.batch_size, 1, self.hidden_size], [self.batch_size, self.hidden_size]])
-            _check_shape_equal(F.shape(value_tensor), "value_tensor", self.cls_name,
-                               [[self.batch_size, 1, self.hidden_size], [self.batch_size, self.hidden_size]])
-            if attention_mask is not None:
-                _check_shape_equal(F.shape(attention_mask), "attention_mask", self.cls_name,
-                                   [[self.batch_size, 1, self.tgt_seq_length], [self.batch_size, self.hidden_size]])
-
         _check_input_dtype(F.dtype(query_tensor), "query_tensor", [mstype.float32, mstype.float16], self.cls_name)
         _check_input_dtype(F.dtype(key_tensor), "key_tensor", [mstype.float32, mstype.float16], self.cls_name)
         _check_input_dtype(F.dtype(value_tensor), "value_tensor", [mstype.float32, mstype.float16], self.cls_name)
@@ -1264,13 +1238,8 @@ class MultiHeadAttention(Cell):
         _check_past_none_input_none(self.use_past, "batch_valid_length", self.cls_name, None,
                                     batch_valid_length_is_tensor, batch_is_default)
         if self.use_past:
-            _check_shape_equal(F.shape(key_past), "key_past", self.cls_name,
-                               [self.batch_size, self.n_head, self.size_per_head, self.tgt_seq_length])
             _check_input_dtype(F.dtype(key_past), "key_past", [mstype.float16], self.cls_name)
-            _check_shape_equal(F.shape(value_past), "value_past", self.cls_name,
-                               [self.batch_size, self.n_head, self.tgt_seq_length, self.size_per_head])
             _check_input_dtype(F.dtype(value_past), "value_past", [mstype.float16], self.cls_name)
-            _check_shape_equal(F.shape(batch_valid_length), "batch_valid_length", self.cls_name, [self.batch_size])
             _check_input_dtype(F.dtype(batch_valid_length), "batch_valid_length", [mstype.int32], self.cls_name)
         return True
 
@@ -1528,7 +1497,7 @@ class TransformerEncoderLayer(Cell):
         if batch_size or use_past:
             Validator.check_positive_int(batch_size)
         self.batch_size = batch_size
-        if _get_parallel_mode() in (ParallelMode.AUTO_PARALLEL,) and _is_sharding_propagation():
+        if _get_parallel_mode() in (ParallelMode.AUTO_PARALLEL,):
             _check_config(parallel_config)
             if num_heads % parallel_config.model_parallel != 0:
                 raise ValueError(
@@ -1770,17 +1739,6 @@ class TransformerEncoderLayer(Cell):
 
     def _check_input(self, x, input_mask, init_reset, batch_valid_length):
         r"""Check inputs"""
-        if not self.use_past or (self.use_past and self.is_first_iteration):
-            _check_shape_equal_without_batch(F.shape(x), "x", self.cls_name,
-                                             [self.seq_length, self.hidden_size])
-            if input_mask is not None:
-                _check_shape_equal(F.shape(input_mask), "input_mask", self.cls_name,
-                                   [F.shape(input_mask)[0], self.seq_length, self.seq_length])
-        else:
-            _check_shape_equal(F.shape(x), "x", self.cls_name, [self.batch_size, 1, self.hidden_size])
-            if input_mask is not None:
-                _check_shape_equal(F.shape(input_mask), "input_mask", self.cls_name,
-                                   [F.shape(input_mask)[0], 1, self.seq_length])
         _check_input_dtype(F.dtype(x), "x", [mstype.float32, mstype.float16], self.cls_name)
         if input_mask is not None:
             _check_input_dtype(F.dtype(input_mask), "input_mask", [mstype.float32, mstype.float16], self.cls_name)
@@ -1795,9 +1753,7 @@ class TransformerEncoderLayer(Cell):
                                     batch_valid_length_is_tensor, batch_is_default)
 
         if self.use_past:
-            _check_shape_equal(F.shape(init_reset), "init_reset", self.cls_name, [1])
             _check_input_dtype(F.dtype(init_reset), "init_reset", [mstype.bool_], self.cls_name)
-            _check_shape_equal(F.shape(batch_valid_length), "batch_valid_length", self.cls_name, [self.batch_size])
             _check_input_dtype(F.dtype(batch_valid_length), "batch_valid_length", [mstype.int32], self.cls_name)
         return True
 
@@ -1933,7 +1889,7 @@ class TransformerDecoderLayer(Cell):
         config_to_attention = parallel_config.dpmp if self.use_moe else parallel_config
         if batch_size or use_past:
             Validator.check_positive_int(batch_size)
-        if _get_parallel_mode() in (ParallelMode.AUTO_PARALLEL,) and _is_sharding_propagation():
+        if _get_parallel_mode() in (ParallelMode.AUTO_PARALLEL,):
             _check_config(parallel_config)
             if num_heads % parallel_config.model_parallel != 0:
                 raise ValueError("For 'TransformerDecoderLayer', the class variable 'num_heads' must be divisibled by "
@@ -2226,31 +2182,14 @@ class TransformerDecoderLayer(Cell):
 
     def _check_input(self, hidden_states, attention_mask, encoder_output, memory_mask, init_reset, batch_valid_length):
         r"""Check inputs"""
-        if not self.use_past or (self.use_past and self.is_first_iteration):
-            _check_shape_equal_without_batch(F.shape(hidden_states), "hidden_states", self.cls_name,
-                                             [self.tgt_seq_length, self.hidden_size])
-            if attention_mask is not None:
-                _check_shape_equal(F.shape(attention_mask), "attention_mask", self.cls_name,
-                                   [F.shape(attention_mask)[0], self.tgt_seq_length, self.tgt_seq_length])
-
-        else:
-            _check_shape_equal(F.shape(hidden_states), "hidden_states", self.cls_name,
-                               [self.batch_size, 1, self.hidden_size])
-            if attention_mask is not None:
-                _check_shape_equal(F.shape(attention_mask), "attention_mask", self.cls_name,
-                                   [self.batch_size, 1, self.tgt_seq_length])
         _check_input_dtype(F.dtype(hidden_states), "hidden_states", [mstype.float32, mstype.float16], self.cls_name)
         if attention_mask is not None:
             _check_input_dtype(F.dtype(attention_mask), "attention_mask", [mstype.float32, mstype.float16],
                                self.cls_name)
         if encoder_output is not None:
-            _check_shape_equal_without_batch(F.shape(encoder_output), "encoder_output", self.cls_name,
-                                             [self.src_seq_length, self.hidden_size])
             _check_input_dtype(F.dtype(encoder_output), "encoder_output",
                                [mstype.float32, mstype.float16], self.cls_name)
         if memory_mask is not None:
-            _check_shape_equal_without_batch(F.shape(memory_mask), "memory_mask", self.cls_name,
-                                             [self.tgt_seq_length, self.src_seq_length])
             _check_input_dtype(F.dtype(memory_mask), "memory_mask",
                                [mstype.float32, mstype.float16], self.cls_name)
 
@@ -2264,9 +2203,7 @@ class TransformerDecoderLayer(Cell):
                                     batch_valid_length_is_tensor, batch_is_default)
 
         if self.use_past:
-            _check_shape_equal(F.shape(init_reset), "init_reset", self.cls_name, [1])
             _check_input_dtype(F.dtype(init_reset), "init_reset", [mstype.bool_], self.cls_name)
-            _check_shape_equal(F.shape(batch_valid_length), "batch_valid_length", self.cls_name, [self.batch_size])
             _check_input_dtype(F.dtype(batch_valid_length), "batch_valid_length", [mstype.int32], self.cls_name)
         return True
 
@@ -2487,7 +2424,7 @@ class TransformerEncoder(Cell):
         _check_moe_config(moe_config, parallel_config)
         self.use_moe = (moe_config.expert_num > 1)
         config_to_layer = parallel_config.moe_parallel_config if self.use_moe else parallel_config.dp_mp_config
-        if _get_parallel_mode() in (ParallelMode.AUTO_PARALLEL,) and _is_sharding_propagation():
+        if _get_parallel_mode() in (ParallelMode.AUTO_PARALLEL,):
             self.add = P.Add()
             self.aux_loss = Tensor(0.0, mstype.float32)
             self.num_layers = num_layers
@@ -2723,7 +2660,7 @@ class TransformerDecoder(Cell):
         _check_config(parallel_config)
         self.use_moe = (moe_config.expert_num > 1)
         config_to_layer = parallel_config.moe_parallel_config if self.use_moe else parallel_config.dp_mp_config
-        if _get_parallel_mode() in (ParallelMode.AUTO_PARALLEL,) and _is_sharding_propagation():
+        if _get_parallel_mode() in (ParallelMode.AUTO_PARALLEL,):
             self.add = P.Add()
             self.aux_loss = Tensor(0.0, mstype.float32)
             self.num_layers = num_layers
@@ -2827,8 +2764,8 @@ class Transformer(Cell):
         the residual addition before the layer normalization. And the default hidden act is `gelu`.
         The details can be found in `Attention is all you need <https://arxiv.org/pdf/1706.03762v5.pdf>`_.
 
-        Note:
-            This is an experimental interface that is subject to change or deletion.
+        .. warning::
+            This is an experimental API that is subject to change or deletion.
 
         Args:
             hidden_size(int): The hidden size of the input.
@@ -2986,7 +2923,7 @@ class Transformer(Cell):
                  moe_config=default_moe_config,
                  parallel_config=default_transformer_config):
         super(Transformer, self).__init__()
-        if _get_parallel_mode() in (ParallelMode.AUTO_PARALLEL,) and _is_sharding_propagation():
+        if _get_parallel_mode() in (ParallelMode.AUTO_PARALLEL,):
             _check_config(parallel_config)
             self.batch_size = batch_size
             self.hidden_size = hidden_size

mindspore/parallel/_utils.py

@@ -52,9 +52,8 @@ def _is_in_hybrid_parallel_mode():
 
 
 def _is_pynative_parallel():
-    run_mode = context.get_context('mode')
     parallel_mode = context.get_auto_parallel_context('parallel_mode')
-    return run_mode == context.PYNATIVE_MODE and parallel_mode in (
+    return context.get_context('mode') == context.PYNATIVE_MODE and parallel_mode in (
         context.ParallelMode.SEMI_AUTO_PARALLEL, context.ParallelMode.AUTO_PARALLEL)
 
 

mindspore/parallel/algo_parameter_config.py

@@ -229,7 +229,7 @@ def set_algo_parameters(**kwargs):
     """
     Set parameters in the algorithm for parallel strategy searching. See a typical use in
     `test_auto_parallel_resnet.py
-    <https://gitee.com/mindspore/mindspore/blob/r2.0.0-alpha/tests/ut/python/parallel/test_auto_parallel_resnet.py>`_.
+    <https://gitee.com/mindspore/mindspore/blob/r2.0/tests/ut/python/parallel/test_auto_parallel_resnet.py>`_.
 
     Note:
         The attribute name is required. This interface works ONLY in AUTO_PARALLEL mode.

mindspore/parallel/checkpoint_transform.py

@@ -22,8 +22,9 @@ from collections import defaultdict
 import numpy as np
 import mindspore as ms
 from mindspore.parallel._parallel_serialization import _rank_list_for_transform_parallel_checkpoint, \
-    _transform_parallel_checkpoint, _get_device_num_from_strategy, _make_dir, _load_strategy_file, \
-    _extract_layout_map, _extract_src_dst_layout_map, _parameter_not_in_local_stage, _extract_pipeline_stage_num
+    _transform_parallel_checkpoint, _get_device_num_from_strategy, _make_dir, \
+    _extract_layout_map, _extract_src_dst_layout_map, _parameter_not_in_local_stage, _extract_pipeline_stage_num, \
+    _merge_protobuf_strategy, _merge_json_strategy
 
 
 __all__ = ["merge_pipeline_strategys", "rank_list_for_transform", "transform_checkpoint_by_rank",
@@ -33,6 +34,9 @@ __all__ = ["merge_pipeline_strategys", "rank_list_for_transform", "transform_che
 def merge_pipeline_strategys(src_strategy_dirs, dst_strategy_file):
     """
     Merge parallel strategy between all pipeline stages in pipeline parallel mode.
+    For more details about converting distributed Checkpoint, please refer to
+    `Distributed Resilience Training and
+    Inference <https://www.mindspore.cn/tutorials/experts/en/r2.0/parallel/resilience_train_and_predict.html>`_.
 
     Note:
         Strategy file of each pipeline stage should be included in src_strategy_dirs.
@@ -55,32 +59,24 @@ def merge_pipeline_strategys(src_strategy_dirs, dst_strategy_file):
         _make_dir(dst_strategy_dir, "path")
     if not os.path.isdir(src_strategy_dirs):
         raise NotADirectoryError("src_strategy_dirs {} is not a directory.".format(src_strategy_dirs))
-    src_strategy_files = os.path.join(src_strategy_dirs, "*.ckpt")
-    dst_parallel_strategy_map = ms.train.node_strategy_pb2.ParallelStrategyMap()
-    merged_stage = []
-    for src_strategy_file in glob.glob(src_strategy_files):
-        src_parallel_strategy_map = _load_strategy_file(src_strategy_file)
-        strategy_items = src_parallel_strategy_map.parallel_strategy_item
-        layout_items = src_parallel_strategy_map.parallel_layout_item
-        if not strategy_items or not layout_items:
-            raise ValueError("The strategy file {} is empty".format(src_strategy_file))
-        pipeline_stage = strategy_items[0].parallel_strategys.stage
-        if pipeline_stage in merged_stage:
-            continue
-        for layout_item in layout_items:
-            layout_item.param_name = "-".join([str(pipeline_stage), layout_item.param_name])
-        dst_parallel_strategy_map.parallel_strategy_item.extend(strategy_items)
-        dst_parallel_strategy_map.parallel_layout_item.extend(layout_items)
-        merged_stage.append(pipeline_stage)
-    dst_parallel_strategy_map.current_stage = 1
-    with open(dst_strategy_file, "wb") as f:
-        f.write(dst_parallel_strategy_map.SerializeToString())
+    src_strategy_files_protobuf = glob.glob(os.path.join(src_strategy_dirs, "*.ckpt"))
+    src_strategy_files_json = glob.glob(os.path.join(src_strategy_dirs, "*.json"))
+    if src_strategy_files_protobuf and src_strategy_files_json:
+        raise ValueError("The strategys format should be all '.ckpt' or all '.json'")
+    is_protobuf = len(src_strategy_files_protobuf) > 0
+    if is_protobuf:
+        _merge_protobuf_strategy(src_strategy_files_protobuf, dst_strategy_file)
+    else:
+        _merge_json_strategy(src_strategy_files_json, dst_strategy_file)
+
 
 
 def rank_list_for_transform(rank_id, src_strategy_file=None, dst_strategy_file=None):
     """
     List of original distributed checkpoint rank index for obtaining the target checkpoint of a rank_id
-    during the distributed checkpoint conversion.
+    during the distributed checkpoint conversion. For more details about converting distributed Checkpoint,
+    please refer to `Distributed Resilience Training and
+    Inference <https://www.mindspore.cn/tutorials/experts/en/r2.0/parallel/resilience_train_and_predict.html>`_.
 
     Args:
         rank_id (int): The rank of which distributed checkpoint needs to be obtained after conversion.
@@ -106,7 +102,7 @@ def rank_list_for_transform(rank_id, src_strategy_file=None, dst_strategy_file=N
         >>> rank_list = rank_list_for_transform(rank_id, "./src_strategy.ckpt", "./dst_strategy.ckpt")
         >>> checkpoint_files_map = {}
         >>> for rank in rank_list:
-        >>>     checkpoint_files_map[rank] = "./pangu{}-100_2.ckpt".format(rank)
+        ...     checkpoint_files_map[rank] = "./pangu{}-100_2.ckpt".format(rank)
 
     """
     if not isinstance(rank_id, int):
@@ -133,14 +129,18 @@ def rank_list_for_transform(rank_id, src_strategy_file=None, dst_strategy_file=N
             src_rank_id_start = src_pipeline_stage_id * src_stage_device_num
             result_set.update([src_rank_id_start + rank for rank in needed_rank_list_in_local_stage])
             handled_pipeline_stage.append(src_pipeline_stage_id)
-    return list(result_set)
+    result_list = list(result_set)
+    result_list.sort(reverse=True)
+    return result_list
 
 
 def transform_checkpoint_by_rank(rank_id, checkpoint_files_map, save_checkpoint_file_name,
                                  src_strategy_file=None, dst_strategy_file=None):
     """
     Transform distributed checkpoint from source sharding strategy to destination sharding strategy by rank
-    for a network.
+    for a network. For more details about converting distributed Checkpoint, please refer to
+    `Distributed Resilience Training and
+    Inference <https://www.mindspore.cn/tutorials/experts/en/r2.0/parallel/resilience_train_and_predict.html>`_.
 
     Args:
         rank_id (int): The rank of which distributed checkpoint needs to be obtained after conversion.
@@ -167,13 +167,13 @@ def transform_checkpoint_by_rank(rank_id, checkpoint_files_map, save_checkpoint_
 
     Examples:
         >>> dst_device_num = 8
-        >>> for rank_id in range(dst_device_num)
-        >>>     rank_list = rank_list_for_transform(rank_id, "./src_strategy.ckpt", "./dst_strategy.ckpt")
-        >>>     checkpoint_files_map = {}
-        >>>     for rank in rank_list:
-        >>>         checkpoint_files_map[rank] = "./origin_checkpoint_rank{}/pangu{}-100_2.ckpt".format(rank)
-        >>>     save_checkpoint_file_name = "./new_checkpoint_rank{}/pangu{}-100_2.ckpt".format(rank_id)
-        >>>     transform_checkpoint_by_rank(rank_id, checkpoint_files_map, save_checkpoint_file_name,
+        >>> for rank_id in range(dst_device_num):
+        ...     rank_list = rank_list_for_transform(rank_id, "./src_strategy.ckpt", "./dst_strategy.ckpt")
+        ...     checkpoint_files_map = {}
+        ...     for rank in rank_list:
+        ...         checkpoint_files_map[rank] = "./origin_checkpoint_rank{}/pangu{}-100_2.ckpt".format(rank)
+        ...     save_checkpoint_file_name = "./new_checkpoint_rank{}/pangu{}-100_2.ckpt".format(rank_id)
+        ...     transform_checkpoint_by_rank(rank_id, checkpoint_files_map, save_checkpoint_file_name,
         ...                                  "./src_strategy.ckpt", "./dst_strategy.ckpt")
 
     """
@@ -222,9 +222,12 @@ def transform_checkpoints(src_checkpoints_dir, dst_checkpoints_dir, ckpt_prefix,
                           dst_strategy_file=None):
     """
     Transform distributed checkpoint from source sharding strategy to destination sharding strategy for a rank.
+    For more details about converting distributed Checkpoint, please refer to
+    `Distributed Resilience Training and
+    Inference <https://www.mindspore.cn/tutorials/experts/en/r2.0/parallel/resilience_train_and_predict.html>`_.
 
     Note:
-        The src_checkpoints_dir directory structure should be organized like "src_checkpoints_dir/rank_0/a.ckpt", the
+        The `src_checkpoints_dir` directory structure should be organized like "src_checkpoints_dir/rank_0/a.ckpt", the
         rank number should be set to a subdirectory and the checkpoint file is stored in this subdirectory. If multiple
         files exist in a rank directory, the last file in the lexicgraphic order would be selected.
 

mindspore/parallel/shard.py

@@ -154,7 +154,7 @@ def shard(fn, in_strategy, out_strategy=None, parameter_plan=None, device="Ascen
     in_strategy and out_strategy define the input and output layout respectively.
     in_strategy/out_strategy should be a tuple, each element of which corresponds to the desired layout of
     this input/output, and None represents data_parallel,
-    which can refer to the description of `mindspore.ops.Primitive.shard`.
+    which can refer to the description of :func:`mindspore.ops.Primitive.shard`.
     The parallel strategies of remaining operators are derived from the strategy specified by the input and output.
 
     Note:
@@ -162,16 +162,18 @@ def shard(fn, in_strategy, out_strategy=None, parameter_plan=None, device="Ascen
         set the parallel mode in `set_auto_parallel_context` to "auto_parallel"
         and the search mode to "sharding_propagation".
         If the input contain Parameter, its strategy should be set in `in_strategy`.
+        For more details about shard, please refer to `Functional Operator Sharding
+        <https://www.mindspore.cn/tutorials/experts/en/r2.0/parallel/pynative_shard_function_parallel.html>`_.
 
     Args:
         fn (Union[Cell, Function]): Function to be executed in parallel.
                                     Its arguments and return value must be Tensor or Parameter.
-                                    If fn is a Cell with parameters, fn needs to be an instantiated object,
+                                    If `fn` is a Cell with parameters, `fn` needs to be an instantiated object,
                                     otherwise its arguments cannot be accessed.
         in_strategy (tuple): Define the layout of inputs, each element of the tuple should be a tuple or None.
                              Tuple defines the layout of the corresponding input
                              and None represents a data parallel strategy.
-        out_strategy (Union[tuple, None]): Define the layout of outputs similar with in_strategy.
+        out_strategy (Union[tuple, None]): Define the layout of outputs similar with `in_strategy`.
                                            It is not in use right now. Default: None.
         parameter_plan (Union[dict, None]): Define the layout for the specified parameters. Each element in dict
                                             defines the layout of the parameter like "param_name: layout".
@@ -180,7 +182,7 @@ def shard(fn, in_strategy, out_strategy=None, parameter_plan=None, device="Ascen
                                             If the parameter name is incorrect or the corresponding parameter
                                             has been set, the parameter setting will be ignored.
                                             Default: None.
-        device (string): Select a certain device target. It is not in use right now.
+        device (string): Select a certain `device` target. It is not in use right now.
                          Support ["CPU", "GPU", "Ascend"]. Default: "Ascend".
         level (int): Option for parallel strategy infer algorithm, namely the object function, maximize computation
                      over communication ratio, maximize speed performance, minimize memory usage etc. It is not in
@@ -190,20 +192,17 @@ def shard(fn, in_strategy, out_strategy=None, parameter_plan=None, device="Ascen
         Function, return the function that will be executed under auto parallel process.
 
     Raises:
-        AssertionError:
-            - If execute mode is not PYNATIVE_MODE.
-            - If parallel mode is not "auto_parallel".
-            - If search_mode it not "sharding_propagation".
-            - If device_target it not "Ascend" or "GPU".
-
-        TypeError:
-            - If `in_strategy` is not a tuple.
-            - If `out_strategy` is not a tuple or None.
-            - If `parameter_plan` is not a dict or None.
-            - If any key in `parameter_plan` is not a str.
-            - If any value in `parameter_plan` is not a tuple.
-            - If `device` is not a str.
-            - If `level` is not a integer.
+        AssertionError: If execute mode is not PYNATIVE_MODE.
+        AssertionError: If parallel mode is not "auto_parallel".
+        AssertionError: If search_mode it not "sharding_propagation".
+        AssertionError: If device_target it not "Ascend" or "GPU".
+        TypeError: If `in_strategy` is not a tuple.
+        TypeError: If `out_strategy` is not a tuple or None.
+        TypeError: If `parameter_plan` is not a dict or None.
+        TypeError: If any key in `parameter_plan` is not a str.
+        TypeError: If any value in `parameter_plan` is not a tuple.
+        TypeError: If `device` is not a str.
+        TypeError: If `level` is not an integer.
 
     Supported Platforms:
         ``Ascend`` ``GPU``

mindspore/profiler/common/validator/validate_path.py

@@ -32,11 +32,11 @@ def check_valid_character_of_path(file_path):
     Returns:
         bool, whether valid.
     """
-    re_path = r'^[/\\_a-zA-Z0-9-_.]+$'
+    re_path = r'^[/\\_a-zA-Z0-9-_.@]+$'
     path_valid = re.fullmatch(re_path, file_path)
     if not path_valid:
         msg = "The output path of profiler only supports alphabets(a-zA-Z), " \
-              "digit(0-9) or {'-', '_', '.', '/'}, but got the absolute path= " + file_path
+              "digit(0-9) or {'-', '_', '.', '/', '@'}, but got the absolute path= " + file_path
         raise RuntimeError(msg)