mindspore 2.2.14__cp39-cp39-win_amd64.whl → 2.4.0__cp39-cp39-win_amd64.whl

mindspore/parallel/_tensor.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Huawei Technologies Co., Ltd
+# Copyright 2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,8 +16,8 @@
 from __future__ import division
 from __future__ import absolute_import
 
+import copy
 import numpy as np
-from mindspore.common import dtype as mstype
 from mindspore.common.tensor import Tensor
 from mindspore.communication.management import get_rank, get_group_size
 from mindspore._c_expression import TensorTransform
@@ -198,7 +198,7 @@ def _get_slice_index(dev_mat, tensor_map, opt_shard_group):
     return tensor_slice_index
 
 
-def _load_tensor(tensor, dev_mat, tensor_map, rank_id=-1):
+def _load_tensor(tensor, dev_mat, tensor_map, full_shape=None, rank_id=-1):
     """
     Get the tensor slice of the local device by the device matrix and the tensor map
 
@@ -214,7 +214,8 @@ def _load_tensor(tensor, dev_mat, tensor_map, rank_id=-1):
         >>> tensor = Tensor(np.ones([32, 32]))
         >>> dev_mat = [2, 4]
         >>> tensor_map = [1, -1]
-        >>> tensor_slice = _load_tensor(tensor, dev_mat, tensor_map)
+        >>> full_shape = [32, 32]
+        >>> tensor_slice = _load_tensor(tensor, dev_mat, tensor_map, full_shape)
     """
     if rank_id == -1:
         rank = get_rank()
@@ -222,11 +223,9 @@ def _load_tensor(tensor, dev_mat, tensor_map, rank_id=-1):
         rank = rank_id
     tensor_strategy = _get_tensor_strategy(dev_mat, tensor_map)
     tensor_slice_index = _get_tensor_slice_index(dev_mat, tensor_strategy, tensor_map, rank)
-    if tensor.dtype == mstype.bfloat16:
-        from mindspore.ops.operations import Cast
-        cpu_cast = Cast().set_device("CPU")
-        tensor = cpu_cast(tensor, mstype.float32)
     np_tensor = tensor.asnumpy()
+    if full_shape:
+        np_tensor = np_tensor.reshape(full_shape)
     np_tensor_list = _chunk_tensor_by_strategy(np_tensor, tensor_strategy)
     np_tensor_slice = np_tensor_list[int(tensor_slice_index)]
     return np_tensor_slice
@@ -249,21 +248,29 @@ def _load_tensor_by_layout(tensor, layout, rank_id):
     """
     if not isinstance(layout, tuple):
         raise TypeError("The layout should be tuple! layout is {}".format(layout))
-    if len(layout) < 6:
-        raise ValueError("The length of layout must be larger than 5! layout is {}".format(layout))
+    if len(layout) < 7:
+        raise ValueError("The length of layout must be larger than 6! layout is {}".format(layout))
     dev_mat = layout[0]
     tensor_map = layout[1]
+    slice_shape = layout[2]
     if not tensor_map:
         return tensor
     uniform_split = layout[4]
     group = layout[5]
+    full_shape = layout[6]
     if uniform_split == 0:
         raise RuntimeError("The load tensor only support uniform split now")
-    tensor_slice = _load_tensor(tensor, dev_mat, tensor_map, rank_id)
+    tensor_slice = _load_tensor(tensor, dev_mat, tensor_map, full_shape, rank_id)
+    if tensor_slice.shape != slice_shape and not group:
+        tensor_slice = tensor_slice.reshape(slice_shape)
     if group:
         # get a totally shard tensor slice for parallel optimizer
         rank = get_rank(group)
         size = get_group_size(group)
+        if tensor_slice.shape != tuple(slice_shape) and slice_shape:
+            slice_shape_extend = copy.deepcopy(slice_shape)
+            slice_shape_extend[0] = slice_shape[0] * size
+            tensor_slice = tensor_slice.reshape(slice_shape_extend)
         tensor_slice = np.split(tensor_slice, size)[rank]
     return Tensor(tensor_slice, tensor.dtype)
 
@@ -327,8 +334,10 @@ def _extract_layout_item(layout_item):
     tensor_map = layout_item[1]
     opt_shard_step = layout_item[4]
     opt_shard_size = layout_item[5]
+    tensor_strategy = _get_tensor_strategy(dev_matrix, tensor_map)
+    model_parallel_shard_size = np.prod(tensor_strategy)
     if opt_shard_size == -1:
-        opt_shard_size = np.prod(dev_matrix) // opt_shard_step
+        opt_shard_size = np.prod(dev_matrix) // model_parallel_shard_size
     return dev_matrix, tensor_map, opt_shard_step, opt_shard_size
 
 
@@ -399,12 +408,35 @@ def _construct_tensor_layout_for_opt_shard(dev_matrix, tensor_map, opt_shard_ste
     if opt_shard_step == 0 or opt_shard_size == 0:
         return dev_matrix, tensor_map, list(origin_full_tensor_shape)
     tensor_strategy = _get_tensor_strategy(dev_matrix, tensor_map)
-    model_parallel_shard_size = np.prod(tensor_strategy)
-    if model_parallel_shard_size != opt_shard_step:
+    repeated_dim = []
+    dev_sharded_index = []
+    for dim in tensor_map:
+        if dim != -1:
+            dev_sharded_index.append(len(dev_matrix) - dim - 1)
+    for index, value in enumerate(dev_matrix):
+        if index not in dev_sharded_index and value > 1:
+            repeated_dim.append(index)
+    if not repeated_dim:
+        raise ValueError("The device_matrix {} and tensor_map {} cannot sharding opt_shard".
+                         format(dev_matrix, tensor_map))
+    if len(repeated_dim) == 1 and np.prod(dev_matrix[repeated_dim[0] + 1:]) != opt_shard_step:
         raise ValueError("The optimizer sharding step {} is not equal to the model parallel sharding size {}.".
-                         format(opt_shard_step, model_parallel_shard_size))
-
+                         format(opt_shard_step, np.prod(dev_matrix[repeated_dim[0] + 1:])))
     first_dim_no_sharding_size = origin_full_tensor_shape[0] // tensor_strategy[0]
+    if (len(repeated_dim) < len(dev_matrix) and len(repeated_dim) > 1) or repeated_dim[0] > 0:
+        tensor_shape_new = list(origin_full_tensor_shape)
+        tensor_shape_new[0] = tensor_strategy[0]
+        accu_shp = 1
+        for i in range(len(repeated_dim) - 1):
+            opt_sharding_size = dev_matrix[repeated_dim[i]]
+            tensor_shape_new.insert(i + 1, opt_sharding_size)
+            accu_shp = accu_shp * opt_sharding_size
+        tensor_shape_new.insert(len(repeated_dim), first_dim_no_sharding_size // accu_shp)
+        tensor_map_new = list(copy.deepcopy(tensor_map))
+        for index, r_dim in enumerate(repeated_dim):
+            tensor_map_new.insert(index + 1, len(dev_matrix) - r_dim - 1)
+        return list(dev_matrix), tensor_map_new, tensor_shape_new
+
     full_tensor_shape = list(origin_full_tensor_shape)
     full_tensor_shape[0] = tensor_strategy[0]
     full_tensor_shape.insert(1, first_dim_no_sharding_size)
@@ -445,7 +477,7 @@ def _get_needed_rank_transform_operator_map_by_layouts(from_tensor_layout, to_te
     result_map = {self_rank: transform_operators}
     for operators in transform_operators:
         op_name = operators[0]
-        if op_name == "AllGather":
+        if op_name == "AllConcat":
             groups = operators[1][:-1]
             stack.append((index, groups))
             index += 1
@@ -459,7 +491,7 @@ def _get_needed_rank_transform_operator_map_by_layouts(from_tensor_layout, to_te
                 index = 0
                 for operators in new_transform_operators:
                     op_name = operators[0]
-                    if op_name == "AllGather" and index < group_info[0]:
+                    if op_name == "AllConcat" and index < group_info[0]:
                         groups = operators[1][:-1]
                         stack.insert(0, (index, groups))
                         index += 1
@@ -484,7 +516,7 @@ def _generate_transform_operator_stack(transform_operators_map, self_rank):
         level = queue_front[1]
         current_operator = queue_front[2]
         if level >= 1:
-            if current_operator[0] == "AllGather":
+            if current_operator[0] == "AllConcat":
                 current_group = current_operator[1][:-1]
                 for rank_id in current_group:
                     handle_queue.append((rank_id, level - 1, transform_operators_map[rank_id][level - 1]))
@@ -516,7 +548,7 @@ def _apply_tensor_transform_operators(transform_operator_stack, tensor_dict, dev
                 if operator[0] != op_name:
                     raise ValueError("The operator in the same level should be equal in the transform tensor operator "
                                      "list, but the find {} and {} in level {}".format(op_name, operator[0], cur_level))
-                if operator[0] != "AllGather":
+                if operator[0] != "AllConcat":
                     tensor_dict[rank_id % device_num] = _apply_operator(operator[0])(tensor_dict[rank_id % device_num],
                                                                                      operator)
                     continue
@@ -525,7 +557,7 @@ def _apply_tensor_transform_operators(transform_operator_stack, tensor_dict, dev
                         raise ValueError("The checkpoint file of rank {} is missing.".format(rank % device_num))
                 allgather_list = [tensor_dict[rank % device_num] for rank in operator[1][:-1]]
                 tmp_tensor_dict[rank_id % device_num] = _apply_operator(operator[0])(allgather_list, operator)
-            if op_name == "AllGather":
+            if op_name == "AllConcat":
                 for rank, value in tmp_tensor_dict.items():
                     tensor_dict[rank % device_num] = value
             level_operators.clear()
@@ -614,7 +646,7 @@ def _apply_operator(operator_name):
         return numpy_data[slice_index]
 
     _apply_operator_map = {"Reshape": _apply_reshape_operator, "StridedSlice": _apply_slice_operator,
-                           "AllGather": _apply_allconcat_operator}
+                           "AllConcat": _apply_allconcat_operator}
     return _apply_operator_map.get(operator_name)
 
 
@@ -651,3 +683,48 @@ def _reshape_param_data_with_weight(param_data, dev_mat, field_size):
     for i in range(1, len(tensor_slices_col)):
         new_tensor = np.concatenate((new_tensor, np.array(tensor_slices_col[i]).reshape(-1, 1)), axis=1)
     return Tensor(new_tensor)
+
+
+def _load_tensor_shape(dev_mat, tensor_map, full_shape=None, rank_id=-1):
+    """get tensor shape by slice"""
+    if rank_id == -1:
+        rank = get_rank()
+    else:
+        rank = rank_id
+    tensor_strategy = _get_tensor_strategy(dev_mat, tensor_map)
+    tensor_slice_index = _get_tensor_slice_index(dev_mat, tensor_strategy, tensor_map, rank)
+    np_tensor_list = _chunk_shape_by_strategy(full_shape, tensor_strategy)
+    np_tensor_slice_index = np_tensor_list[int(tensor_slice_index)]
+    res = []
+    for index in np_tensor_slice_index:
+        res.append(slice(index[0], index[1]))
+    return tuple(res)
+
+
+def _chunk_shape_by_strategy(full_shape, strategy):
+    """chunk shape by strategy"""
+    shape = []
+    for i in full_shape:
+        shape.append([0, i])
+    return _chunk_shape(shape, strategy, len(strategy))
+
+
+def _chunk_shape(np_tensor, strategy, depth):
+    """_chunk shape"""
+    output = []
+    axis = len(np_tensor) - depth
+    left, right = np_tensor[axis]
+    num = strategy[0]
+    chunk_size = (right - left) / num
+    append = [[i, int(i + chunk_size)] for i in range(left, right) if i % chunk_size == 0]
+    np_tensor_new = []
+    for i in append:
+        np_tensor_tmp = copy.deepcopy(np_tensor)
+        np_tensor_tmp[axis] = i
+        np_tensor_new.append(np_tensor_tmp)
+    if depth == 1:
+        return np_tensor_new
+    for ret_ in np_tensor_new:
+        output.extend(
+            _chunk_shape(ret_, strategy[len(strategy) - depth + 1:len(strategy)], depth - 1))
+    return output

mindspore/parallel/_transformer/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
+# Copyright 2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mindspore/parallel/_transformer/layers.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
+# Copyright 2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mindspore/parallel/_transformer/loss.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
+# Copyright 2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mindspore/parallel/_transformer/moe.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
+# Copyright 2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mindspore/parallel/_transformer/op_parallel_config.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
+# Copyright 2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mindspore/parallel/_transformer/transformer.py

@@ -1,4 +1,4 @@
-# Copyright 2021-2022 Huawei Technologies Co., Ltd
+# Copyright 2021-2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -400,7 +400,7 @@ class FeedForward(Cell):
             >>> from mindspore.nn.transformer import FeedForward
             >>> from mindspore import dtype as mstype
             >>> from mindspore import Tensor, nn
-            >>> import mindspore.ops as ops
+            >>> from mindspore import ops
             >>> model = FeedForward(hidden_size=15, ffn_hidden_size=30, dropout_rate=0.1)
             >>> tensor = Tensor(np.ones((2, 20, 15)), mstype.float32)
             >>> output = model(tensor)

mindspore/parallel/_utils.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Huawei Technologies Co., Ltd
+# Copyright 2023-2024 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,6 +13,7 @@
 # limitations under the License.
 # ============================================================================
 """Utils of auto parallel"""
+import os
 from importlib import import_module
 import numpy as np
 import mindspore as ms
@@ -22,11 +23,14 @@ from mindspore.common.tensor import Tensor
 from mindspore.common.dtype import dtype_to_nptype
 from mindspore.common import dtype as mstype
 from mindspore.communication.management import get_group_size, get_rank
+from mindspore.communication._comm_helper import _is_initialized
 from mindspore.parallel._auto_parallel_context import auto_parallel_context
 from mindspore.common.seed import get_seed
 from mindspore._c_expression import GraphExecutor_
 from mindspore.parallel._tensor import _load_tensor_by_layout
 
+SUPPORTED_TUPLE_IN_TUPLE_STRATEGY = ["GroupedMatmul", "FusedInferAttentionScore", "Custom"]
+
 
 def _get_parallel_mode():
     """Get parallel mode."""
@@ -43,6 +47,16 @@ def _is_in_auto_parallel_mode():
     return _get_parallel_mode() in [ms.ParallelMode.SEMI_AUTO_PARALLEL, ms.ParallelMode.AUTO_PARALLEL]
 
 
+def _is_parallel_mode():
+    if not _is_initialized() or context.get_context('mode') == context.PYNATIVE_MODE:
+        return False
+    if os.getenv("RUN_MODE") != "predict":
+        return False
+    if get_group_size() > 1 and _get_parallel_mode() == ms.ParallelMode.STAND_ALONE:
+        return True
+    return False
+
+
 def _is_in_data_parallel_mode():
     return _get_parallel_mode() == ms.ParallelMode.DATA_PARALLEL
 
@@ -92,6 +106,18 @@ def _need_to_full():
 
 def _slice_parameter(parameter, phase, layout):
     """Slice python parameter obj according to the layout."""
+    is_train_phase = phase.startswith('train')
+    is_prefill_phase = phase.startswith('prefill')
+    if layout is not None and parameter.from_ckpt and not is_train_phase:
+        is_opt_shard_group = layout[5]
+        if not parameter.sliced and is_prefill_phase and is_opt_shard_group:
+            rank = get_rank()
+            new_tensor = _load_tensor_by_layout(parameter, layout, rank)
+            parameter.set_data(new_tensor, True)
+            return
+        layout_shape = layout[2]
+        parameter.shape = tuple(layout_shape)
+        return
     graph_executor = GraphExecutor_.get_instance()
     new_param = parameter.init_data(layout, set_sliced=True)
     parameter = new_param
@@ -135,7 +161,7 @@ def _to_full_shapes(shapes, device_num):
             if len(shape) != len(dataset_strategy[index]):
                 raise ValueError("The input shapes item size {} is not equal to "
                                  "dataset strategy item size {}".format(len(shape), len(dataset_strategy[index])))
-            new_shape = ()
+            new_shape = []
             for i, item in enumerate(shape):
                 if item > 0:
                     new_shape += (item * dataset_strategy[index][i],)  # static shape
@@ -144,16 +170,69 @@ def _to_full_shapes(shapes, device_num):
             new_shapes.append(new_shape)
         return new_shapes
     for shape in shapes:
-        new_shape = ()
+        shape_v = []
         for i, item in enumerate(shape):
             if i == 0 and item > 0:
-                new_shape += (item * device_num,)  # only for static shape
+                shape_v += (item * device_num,)  # only for static shape
             else:
-                new_shape += (item,)
-        new_shapes.append(new_shape)
+                shape_v += (item,)
+        new_shapes.append(shape_v)
     return new_shapes
 
 
+def _origin_shapes(shapes):
+    """resume origin shape after full shape."""
+    if _need_to_full():
+        device_num = _get_device_num() // _get_pipeline_stages()
+    else:
+        return shapes
+    new_shapes = []
+    dataset_strategy = ()
+    if context.get_auto_parallel_context("dataset_strategy") not in ("data_parallel", "full_batch"):
+        dataset_strategy = context.get_auto_parallel_context("dataset_strategy")
+    if dataset_strategy:
+        if len(shapes) != len(dataset_strategy):
+            raise ValueError("The input shapes size {} is not equal to "
+                             "dataset strategy size {}".format(len(shapes), len(dataset_strategy)))
+        for index, shape in enumerate(shapes):
+            if len(shape) != len(dataset_strategy[index]):
+                raise ValueError("The input shapes item size {} is not equal to "
+                                 "dataset strategy item size {}".format(len(shape), len(dataset_strategy[index])))
+            new_shape = []
+            for i, item in enumerate(shape):
+                if item > 0:
+                    new_shape += (item // dataset_strategy[index][i],)  # static shape
+                else:
+                    new_shape += (item,)  # dynamic shape
+            new_shapes.append(new_shape)
+        return new_shapes
+    for shape in shapes:
+        shape_v = []
+        for i, item in enumerate(shape):
+            if i == 0 and item > 0:
+                shape_v += (item // device_num,)  # only for static shape
+            else:
+                shape_v += (item,)
+        new_shapes.append(shape_v)
+    return new_shapes
+
+
+def _dynamic_shape_for_dataset(dataset_shapes, dynamic_shapes):
+    """convert static dataset shapes to dynamic shape"""
+    if len(dataset_shapes) != len(dynamic_shapes):
+        raise ValueError("The dataset shapes size of {} is not equal to "
+                         "dynamic shapes size of {}".format(dataset_shapes, dynamic_shapes))
+    ret = dataset_shapes
+    for i in range(len(dynamic_shapes)):
+        if len(dataset_shapes[i]) != len(dynamic_shapes[i]):
+            raise ValueError("The dataset shapes size of {} is not equal to "
+                             "dynamic shapes size of {}".format(dataset_shapes, dynamic_shapes))
+        for j in range(len(dynamic_shapes[i])):
+            if dynamic_shapes[i][j] == -1:
+                ret[i][j] = -1
+    return ret
+
+
 def _to_full_tensor(elem, global_device_num, global_rank, scaling_sens=None):
     """Convert numpy to tensor, expanding batch dimension according to device_num, adapt to feed the data
        from host solution.
@@ -418,6 +497,94 @@ def _infer_rank_list(train_map, predict_map=None):
     return ret
 
 
+def _handle_symbol_inputs(symbol_inputs):
+    """handle symbol inputs"""
+    dataset_strategy = ()
+    divisor_key = "divisor"
+    # dataset strategy is set
+    if context.get_auto_parallel_context("dataset_strategy") not in ("data_parallel", "full_batch"):
+        dataset_strategy = context.get_auto_parallel_context("dataset_strategy")
+    if dataset_strategy:
+        if len(symbol_inputs) != len(dataset_strategy):
+            raise ValueError("The symbol_inputs size {} is not equal to "
+                             "dataset strategy size {}".format(len(symbol_inputs), len(dataset_strategy)))
+        for index, shape in enumerate(symbol_inputs):
+            dataset_ele_s = dataset_strategy[index]
+            if len(shape) != len(dataset_ele_s):
+                raise ValueError("The symbol_inputs item size {} is not equal to "
+                                 "dataset strategy item size {}".format(len(shape), len(dataset_ele_s)))
+
+            for i, item in enumerate(shape):
+                if isinstance(item, dict):  # symbol
+                    symbol_inputs[index][i][divisor_key] = symbol_inputs[index][i][divisor_key] * dataset_ele_s[i]
+                else:  # common shape
+                    symbol_inputs[index][i] = item * dataset_ele_s[i]
+
+        return symbol_inputs
+
+    # full batch is set
+    device_num = _get_device_num() // _get_pipeline_stages()
+    for index, shape in enumerate(symbol_inputs):
+        for i, item in enumerate(shape):
+            if i == 0 and isinstance(item, dict):  # symbol
+                symbol_inputs[index][i][divisor_key] = symbol_inputs[index][i][divisor_key] * device_num
+
+    return symbol_inputs
+
+
+def _no_need_to_change_symbols(shapes):
+    """no need to handle the symbol if full_batch is true or it's not parallel mode"""
+    if not _need_to_full():
+        return True
+
+    # if static shape, return
+    is_dynamic_shape = False
+    for shape in shapes:
+        if any(i < 0 for i in shape):
+            is_dynamic_shape = True
+            break
+    if is_dynamic_shape is False:
+        return True
+
+    return False
+
+
+def _change_symbols_for_parallel(shapes, symbol_inputs=None):
+    """create or modify symbol inputs"""
+    if _no_need_to_change_symbols(shapes) is True:
+        return symbol_inputs
+    # the symbol_inputs is [[{'divisor': 8}, 16], [{'divisor': 8}, 16]]
+    # the dataset_shapes is [(-1, 16), (-1, 16)]
+    # if symbol_inputs is [None, None, ..., None], reset it
+    if symbol_inputs is not None and all(s is None for s in symbol_inputs):
+        symbol_inputs = []
+
+    # if symbol inputs is none or empty, create default symbol inputs
+    # if symbol inputs is not none, handle the empty symbol
+    divisor_key = "divisor"
+    if symbol_inputs is None or bool(symbol_inputs) is False:
+        symbol_inputs = [list(shape) for shape in shapes]  # tuple to list
+        for i, s in enumerate(symbol_inputs):
+            for j, item in enumerate(s):
+                if item == -1:
+                    symbol_inputs[i][j] = {divisor_key: 1}
+    else:
+        for i, s in enumerate(symbol_inputs):
+            # the symbol_inputs may be [None, [{'divisor': 8}, 16]]
+            # and the dataset_shapes is [(-1, 16), (-1, 16)], need to handle None
+            if s is None:
+                symbol_inputs[i] = shapes[i]
+                for k, item in enumerate(symbol_inputs[i]):
+                    if item == -1:
+                        symbol_inputs[i][k] = {divisor_key: 1}
+                s = symbol_inputs[i]
+            for j, item in enumerate(s):
+                if isinstance(item, dict) and bool(item) is False:  # the item is empty
+                    symbol_inputs[i][j] = {divisor_key: 1}
+
+    return _handle_symbol_inputs(symbol_inputs)
+
+
 def _grads_divided_by_device_num_if_recomputation(grads):
     """
     If in pynative parallel and full_batch is True, divide grads by device num to ensure that the gradients is correct.

mindspore/parallel/algo_parameter_config.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Huawei Technologies Co., Ltd
+# Copyright 2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -227,16 +227,14 @@ get_algo_parameters_config_func_map = {
                  enable_algo_approxi=bool, algo_approxi_epsilon=float)
 def set_algo_parameters(**kwargs):
     """
-    Set parameters in the algorithm for parallel strategy searching. See a typical use in
-    `test_auto_parallel_resnet.py
-    <https://gitee.com/mindspore/mindspore/blob/r2.2/tests/ut/python/parallel/test_auto_parallel_resnet.py>`_.
+    Set parameters in the algorithm for parallel strategy searching.
 
     Note:
         The attribute name is required. This interface works ONLY in AUTO_PARALLEL mode.
 
     Args:
         fully_use_devices (bool): Whether ONLY searching strategies that fully use all available devices.
-            Default: ``True`` . For example with 8 devices available, if set ``True`` , strategy (4, 1) will not be
+            Default: ``False`` . For example with 8 devices available, if set ``True`` , strategy (4, 1) will not be
             included in ReLU's candidate strategies, because strategy (4, 1) only utilizes 4 devices.
         elementwise_op_strategy_follow (bool): Whether the elementwise operator has the consistent strategies as its
             subsequent operators. Elementwise operators refer to operators that operate on input element by element,
@@ -266,14 +264,14 @@ def set_algo_parameters(**kwargs):
 
             For the Ascend devices, users need to prepare the rank table, set rank_id and device_id.
             Please see the `rank table startup
-            <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/rank_table.html>`_
+            <https://www.mindspore.cn/docs/en/master/model_train/parallel/rank_table.html>`_
             for more details.
 
             For the GPU devices, users need to prepare the host file and mpi, please see the `mpirun startup
-            <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/mpirun.html>`_ .
+            <https://www.mindspore.cn/docs/en/master/model_train/parallel/mpirun.html>`_ .
 
             For the CPU device, users need to write a dynamic cluster startup script, please see the `Dynamic Cluster
-            Startup <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/dynamic_cluster.html>`_ .
+            Startup <https://www.mindspore.cn/docs/en/master/model_train/parallel/dynamic_cluster.html>`_ .
 
         >>> import numpy as np
         >>> import mindspore as ms
@@ -284,7 +282,7 @@ def set_algo_parameters(**kwargs):
         >>>
         >>> ms.set_context(mode=ms.GRAPH_MODE)
         >>> ms.set_auto_parallel_context(parallel_mode=ms.ParallelMode.AUTO_PARALLEL,
-        >>>                              search_mode="sharding_propagation")
+        ...                              search_mode="sharding_propagation")
         >>> init()
         >>> ms.set_algo_parameters(fully_use_devices=True)
         >>> ms.set_algo_parameters(elementwise_op_strategy_follow=True)
@@ -388,7 +386,7 @@ def reset_algo_parameters():
 
     After reset, the values of the attributes are:
 
-    - fully_use_devices: True.
+    - fully_use_devices: False.
     - elementwise_op_strategy_follow: False.
     - enable_algo_approxi: False.
     - algo_approxi_epsilon: 0.1.