mindspore 2.2.11__cp39-cp39-win_amd64.whl → 2.3.0__cp39-cp39-win_amd64.whl

mindspore/parallel/_transformer/transformer.py

@@ -1,4 +1,4 @@
-# Copyright 2021-2022 Huawei Technologies Co., Ltd
+# Copyright 2021-2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -400,7 +400,7 @@ class FeedForward(Cell):
             >>> from mindspore.nn.transformer import FeedForward
             >>> from mindspore import dtype as mstype
             >>> from mindspore import Tensor, nn
-            >>> import mindspore.ops as ops
+            >>> from mindspore import ops
             >>> model = FeedForward(hidden_size=15, ffn_hidden_size=30, dropout_rate=0.1)
             >>> tensor = Tensor(np.ones((2, 20, 15)), mstype.float32)
             >>> output = model(tensor)
@@ -805,14 +805,14 @@ class MultiHeadAttention(Cell):
             - **attention_mask** (Tensor) - If the use_past is False or is_first_iteration=True, the attention mask
               matrix should ba (batch_size, src_seq_length, tgt_seq_length), or None. None means there will be no mask
               in softmax computation. Otherwise, the mask must be (batch_size, 1, tgt_seq_length)
-            - **key_past** (Tensor) - Float16 tensor with shape (batch_size, num_heads, size_per_head, tgt_seq_length).
+            - **key_past** (Tensor) - float16 tensor with shape (batch_size, num_heads, size_per_head, tgt_seq_length).
               The past calculated key vector. Used for incremental prediction when the use_past is True.
               Default None.
-            - **value_past** (Tensor) - Float16 tensor with shape
+            - **value_past** (Tensor) - float16 tensor with shape
               (batch_size, num_heads, tgt_seq_length, size_per_head).
               The past calculated value vector. Used for incremental prediction when the use_past is True.
               Default None.
-            - **batch_valid_length** (Tensor) - Int32 tensor with shape (batch_size,) the past calculated the index.
+            - **batch_valid_length** (Tensor) - int32 tensor with shape (batch_size,) the past calculated the index.
               Used for incremental prediction when the use_past is True. Default None.
 
         Outputs:
@@ -1412,7 +1412,7 @@ class TransformerEncoderLayer(Cell):
               be no mask in softmax computation. Otherwise, should be [batch_size, 1, hidden_size]
             - **init_reset** (Tensor) - A bool tensor with shape [1], used to clear the past key parameter and
               past value parameter used in the incremental prediction. Only valid when use_past is True. Default True.
-            - **batch_valid_length** (Tensor) - Int32 tensor with shape [batch_size] the past calculated the index.
+            - **batch_valid_length** (Tensor) - int32 tensor with shape [batch_size] the past calculated the index.
               Used for incremental prediction when the use_past is True. Default None.
 
         Outputs:
@@ -1824,7 +1824,7 @@ class TransformerDecoderLayer(Cell):
               means there will be no mask in softmax computation in cross attention. Default None.
             - **init_reset** (Tensor) - A bool tensor with shape [1], used to clear the past key parameter and
               past value parameter used in the incremental prediction. Only valid when use_past is True. Default True.
-            - **batch_valid_length** (Tensor) - Int32 tensor with shape [batch_size] the past calculated the index.
+            - **batch_valid_length** (Tensor) - int32 tensor with shape [batch_size] the past calculated the index.
               Used for incremental prediction when the use_past is True. Default None.
 
         Outputs:
@@ -2333,7 +2333,7 @@ class TransformerEncoder(Cell):
               be no mask in softmax computation. Otherwise, should be [batch_size, 1, hidden_size]
             - **init_reset** (Tensor) - A bool tensor with shape [1], used to clear the past key parameter and
               past value parameter used in the incremental prediction. Only valid when use_past is True. Default True.
-            - **batch_valid_length** (Tensor) - Int32 tensor with shape [batch_size] the past calculated the index.
+            - **batch_valid_length** (Tensor) - int32 tensor with shape [batch_size] the past calculated the index.
               Used for incremental prediction when the use_past is True. Default None.
 
         Outputs:
@@ -2589,7 +2589,7 @@ class TransformerDecoder(Cell):
               means there will be no mask in softmax computation in cross attention. Default None.
             - **init_reset** (Tensor) - A bool tensor with shape [1], used to clear the past key parameter and
               past value parameter used in the incremental prediction. Only valid when use_past is True. Default True.
-            - **batch_valid_length** (Tensor) - Int32 tensor with shape [batch_size] the past calculated the index.
+            - **batch_valid_length** (Tensor) - int32 tensor with shape [batch_size] the past calculated the index.
               Used for incremental prediction when the use_past is True. Default None.
 
         Outputs:
@@ -2842,7 +2842,7 @@ class Transformer(Cell):
               seq_length, hidden_size], this should be none if the decoder layer is 0 or the user wants no mask.
             - **init_reset** (Tensor) - A bool tensor with shape [1], used to clear the past key parameter and
               past value parameter used in the incremental prediction. Only valid when use_past is True. Default True.
-            - **batch_valid_length** (Tensor) - Int32 tensor with shape [batch_size] the past calculated the index.
+            - **batch_valid_length** (Tensor) - int32 tensor with shape [batch_size] the past calculated the index.
               Used for incremental prediction when the use_past is True. Default None.
 
         Outputs:

mindspore/parallel/_utils.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Huawei Technologies Co., Ltd
+# Copyright 2023-2024 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,6 +27,8 @@ from mindspore.common.seed import get_seed
 from mindspore._c_expression import GraphExecutor_
 from mindspore.parallel._tensor import _load_tensor_by_layout
 
+SUPPORTED_TUPLE_IN_TUPLE_STRATEGY = ["GroupedMatmul", "FusedInferAttentionScore"]
+
 
 def _get_parallel_mode():
     """Get parallel mode."""
@@ -92,6 +94,18 @@ def _need_to_full():
 
 def _slice_parameter(parameter, phase, layout):
     """Slice python parameter obj according to the layout."""
+    is_train_phase = phase.startswith('train')
+    is_prefill_phase = phase.startswith('prefill')
+    if layout is not None and parameter.from_ckpt and not is_train_phase:
+        is_opt_shard_group = layout[5]
+        if not parameter.sliced and is_prefill_phase and is_opt_shard_group:
+            rank = get_rank()
+            new_tensor = _load_tensor_by_layout(parameter, layout, rank)
+            parameter.set_data(new_tensor, True)
+            return
+        layout_shape = layout[2]
+        parameter.shape = tuple(layout_shape)
+        return
     graph_executor = GraphExecutor_.get_instance()
     new_param = parameter.init_data(layout, set_sliced=True)
     parameter = new_param
@@ -135,7 +149,7 @@ def _to_full_shapes(shapes, device_num):
             if len(shape) != len(dataset_strategy[index]):
                 raise ValueError("The input shapes item size {} is not equal to "
                                  "dataset strategy item size {}".format(len(shape), len(dataset_strategy[index])))
-            new_shape = ()
+            new_shape = []
             for i, item in enumerate(shape):
                 if item > 0:
                     new_shape += (item * dataset_strategy[index][i],)  # static shape
@@ -144,16 +158,69 @@ def _to_full_shapes(shapes, device_num):
             new_shapes.append(new_shape)
         return new_shapes
     for shape in shapes:
-        new_shape = ()
+        shape_v = []
+        for i, item in enumerate(shape):
+            if i == 0 and item > 0:
+                shape_v += (item * device_num,)  # only for static shape
+            else:
+                shape_v += (item,)
+        new_shapes.append(shape_v)
+    return new_shapes
+
+
+def _origin_shapes(shapes):
+    """resume origin shape after full shape."""
+    if _need_to_full():
+        device_num = _get_device_num() // _get_pipeline_stages()
+    else:
+        return shapes
+    new_shapes = []
+    dataset_strategy = ()
+    if context.get_auto_parallel_context("dataset_strategy") not in ("data_parallel", "full_batch"):
+        dataset_strategy = context.get_auto_parallel_context("dataset_strategy")
+    if dataset_strategy:
+        if len(shapes) != len(dataset_strategy):
+            raise ValueError("The input shapes size {} is not equal to "
+                             "dataset strategy size {}".format(len(shapes), len(dataset_strategy)))
+        for index, shape in enumerate(shapes):
+            if len(shape) != len(dataset_strategy[index]):
+                raise ValueError("The input shapes item size {} is not equal to "
+                                 "dataset strategy item size {}".format(len(shape), len(dataset_strategy[index])))
+            new_shape = []
+            for i, item in enumerate(shape):
+                if item > 0:
+                    new_shape += (item // dataset_strategy[index][i],)  # static shape
+                else:
+                    new_shape += (item,)  # dynamic shape
+            new_shapes.append(new_shape)
+        return new_shapes
+    for shape in shapes:
+        shape_v = []
         for i, item in enumerate(shape):
             if i == 0 and item > 0:
-                new_shape += (item * device_num,)  # only for static shape
+                shape_v += (item // device_num,)  # only for static shape
             else:
-                new_shape += (item,)
-        new_shapes.append(new_shape)
+                shape_v += (item,)
+        new_shapes.append(shape_v)
     return new_shapes
 
 
+def _dynamic_shape_for_dataset(dataset_shapes, dynamic_shapes):
+    """convert static dataset shapes to dynamic shape"""
+    if len(dataset_shapes) != len(dynamic_shapes):
+        raise ValueError("The dataset shapes size of {} is not equal to "
+                         "dynamic shapes size of {}".format(dataset_shapes, dynamic_shapes))
+    ret = dataset_shapes
+    for i in range(len(dynamic_shapes)):
+        if len(dataset_shapes[i]) != len(dynamic_shapes[i]):
+            raise ValueError("The dataset shapes size of {} is not equal to "
+                             "dynamic shapes size of {}".format(dataset_shapes, dynamic_shapes))
+        for j in range(len(dynamic_shapes[i])):
+            if dynamic_shapes[i][j] == -1:
+                ret[i][j] = -1
+    return ret
+
+
 def _to_full_tensor(elem, global_device_num, global_rank, scaling_sens=None):
     """Convert numpy to tensor, expanding batch dimension according to device_num, adapt to feed the data
        from host solution.
@@ -418,6 +485,94 @@ def _infer_rank_list(train_map, predict_map=None):
     return ret
 
 
+def _handle_symbol_inputs(symbol_inputs):
+    """handle symbol inputs"""
+    dataset_strategy = ()
+    divisor_key = "divisor"
+    # dataset strategy is set
+    if context.get_auto_parallel_context("dataset_strategy") not in ("data_parallel", "full_batch"):
+        dataset_strategy = context.get_auto_parallel_context("dataset_strategy")
+    if dataset_strategy:
+        if len(symbol_inputs) != len(dataset_strategy):
+            raise ValueError("The symbol_inputs size {} is not equal to "
+                             "dataset strategy size {}".format(len(symbol_inputs), len(dataset_strategy)))
+        for index, shape in enumerate(symbol_inputs):
+            dataset_ele_s = dataset_strategy[index]
+            if len(shape) != len(dataset_ele_s):
+                raise ValueError("The symbol_inputs item size {} is not equal to "
+                                 "dataset strategy item size {}".format(len(shape), len(dataset_ele_s)))
+
+            for i, item in enumerate(shape):
+                if isinstance(item, dict):  # symbol
+                    symbol_inputs[index][i][divisor_key] = symbol_inputs[index][i][divisor_key] * dataset_ele_s[i]
+                else:  # common shape
+                    symbol_inputs[index][i] = item * dataset_ele_s[i]
+
+        return symbol_inputs
+
+    # full batch is set
+    device_num = _get_device_num() // _get_pipeline_stages()
+    for index, shape in enumerate(symbol_inputs):
+        for i, item in enumerate(shape):
+            if i == 0 and isinstance(item, dict):  # symbol
+                symbol_inputs[index][i][divisor_key] = symbol_inputs[index][i][divisor_key] * device_num
+
+    return symbol_inputs
+
+
+def _no_need_to_change_symbols(shapes):
+    """no need to handle the symbol if full_batch is true or it's not parallel mode"""
+    if not _need_to_full():
+        return True
+
+    # if static shape, return
+    is_dynamic_shape = False
+    for shape in shapes:
+        if any(i < 0 for i in shape):
+            is_dynamic_shape = True
+            break
+    if is_dynamic_shape is False:
+        return True
+
+    return False
+
+
+def _change_symbols_for_parallel(shapes, symbol_inputs=None):
+    """create or modify symbol inputs"""
+    if _no_need_to_change_symbols(shapes) is True:
+        return symbol_inputs
+    # the symbol_inputs is [[{'divisor': 8}, 16], [{'divisor': 8}, 16]]
+    # the dataset_shapes is [(-1, 16), (-1, 16)]
+    # if symbol_inputs is [None, None, ..., None], reset it
+    if symbol_inputs is not None and all(s is None for s in symbol_inputs):
+        symbol_inputs = []
+
+    # if symbol inputs is none or empty, create default symbol inputs
+    # if symbol inputs is not none, handle the empty symbol
+    divisor_key = "divisor"
+    if symbol_inputs is None or bool(symbol_inputs) is False:
+        symbol_inputs = [list(shape) for shape in shapes]  # tuple to list
+        for i, s in enumerate(symbol_inputs):
+            for j, item in enumerate(s):
+                if item == -1:
+                    symbol_inputs[i][j] = {divisor_key: 1}
+    else:
+        for i, s in enumerate(symbol_inputs):
+            # the symbol_inputs may be [None, [{'divisor': 8}, 16]]
+            # and the dataset_shapes is [(-1, 16), (-1, 16)], need to handle None
+            if s is None:
+                symbol_inputs[i] = shapes[i]
+                for k, item in enumerate(symbol_inputs[i]):
+                    if item == -1:
+                        symbol_inputs[i][k] = {divisor_key: 1}
+                s = symbol_inputs[i]
+            for j, item in enumerate(s):
+                if isinstance(item, dict) and bool(item) is False:  # the item is empty
+                    symbol_inputs[i][j] = {divisor_key: 1}
+
+    return _handle_symbol_inputs(symbol_inputs)
+
+
 def _grads_divided_by_device_num_if_recomputation(grads):
     """
     If in pynative parallel and full_batch is True, divide grads by device num to ensure that the gradients is correct.

mindspore/parallel/algo_parameter_config.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Huawei Technologies Co., Ltd
+# Copyright 2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -227,9 +227,7 @@ get_algo_parameters_config_func_map = {
                  enable_algo_approxi=bool, algo_approxi_epsilon=float)
 def set_algo_parameters(**kwargs):
     """
-    Set parameters in the algorithm for parallel strategy searching. See a typical use in
-    `test_auto_parallel_resnet.py
-    <https://gitee.com/mindspore/mindspore/blob/r2.2/tests/ut/python/parallel/test_auto_parallel_resnet.py>`_.
+    Set parameters in the algorithm for parallel strategy searching.
 
     Note:
         The attribute name is required. This interface works ONLY in AUTO_PARALLEL mode.
@@ -266,14 +264,14 @@ def set_algo_parameters(**kwargs):
 
             For the Ascend devices, users need to prepare the rank table, set rank_id and device_id.
             Please see the `rank table startup
-            <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/rank_table.html>`_
+            <https://www.mindspore.cn/tutorials/experts/en/master/parallel/rank_table.html>`_
             for more details.
 
             For the GPU devices, users need to prepare the host file and mpi, please see the `mpirun startup
-            <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/mpirun.html>`_ .
+            <https://www.mindspore.cn/tutorials/experts/en/master/parallel/mpirun.html>`_ .
 
             For the CPU device, users need to write a dynamic cluster startup script, please see the `Dynamic Cluster
-            Startup <https://www.mindspore.cn/tutorials/experts/en/r2.2/parallel/dynamic_cluster.html>`_ .
+            Startup <https://www.mindspore.cn/tutorials/experts/en/master/parallel/dynamic_cluster.html>`_ .
 
         >>> import numpy as np
         >>> import mindspore as ms
@@ -284,7 +282,7 @@ def set_algo_parameters(**kwargs):
         >>>
         >>> ms.set_context(mode=ms.GRAPH_MODE)
         >>> ms.set_auto_parallel_context(parallel_mode=ms.ParallelMode.AUTO_PARALLEL,
-        >>>                              search_mode="sharding_propagation")
+        ...                              search_mode="sharding_propagation")
         >>> init()
         >>> ms.set_algo_parameters(fully_use_devices=True)
         >>> ms.set_algo_parameters(elementwise_op_strategy_follow=True)