mindspore 2.2.14__cp39-cp39-win_amd64.whl → 2.3.0__cp39-cp39-win_amd64.whl

mindspore/parallel/_dp_allreduce_fusion.py

@@ -1,159 +1,159 @@
-# Copyright 2020 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""Data parallel allreduce fusion"""
-from __future__ import absolute_import
-from __future__ import division
-
-import ctypes
-
-_MAX_GROUP_NAME_LEN = 127
-_HCCL_LIB = 'libhccl.so'
-
-
-def _load_lib():
-    try:
-        hccl_lib = ctypes.CDLL(_HCCL_LIB)
-    except Exception:
-        raise RuntimeError('Get hccl lib error')
-
-    return hccl_lib
-
-
-def _c_str(string):
-    """Convert a python string to C string."""
-    if not isinstance(string, str):
-        string = string.decode('ascii')
-    return ctypes.c_char_p(string.encode('utf-8'))
-
-
-def _c_array(ctype, values):
-    """Create ctypes array from a python array."""
-    return (ctype * len(values))(*values)
-
-
-def _set_fusion_strategy_by_idx(idx_list, group="hccl_world_group"):
-    """
-    A function set gradient segment strategy according to the index list.
-
-    Note:
-        In the back propagation,
-        the fusion of the allreduce operators with a fusion attribute equals 1,
-        will be performed according to the idx_list,
-        to achieve the effect of parallel between calculation and communication.
-
-    Args:
-        idx_list (list): The index list of the gradient.
-        group (str): The hccl communication group.
-
-    Raises:
-        TypeError: If group is not a python str.
-        TypeError: If idx_list is not a python list.
-        TypeError: If type of idx_list item is not int.
-        ValueError: If group name length is out of range.
-        ValueError: If idx_list length is 0.
-        ValueError: If idx_list item is less than 0.
-        RuntimeError: If allreduce split failed.
-    """
-    try:
-        lib_ctype = _load_lib()
-    except RuntimeError:
-        import hccl_test.manage.api as hccl
-        hccl.set_fusion_strategy_by_idx()
-        return
-    finally:
-        pass
-    if isinstance(group, str):
-        group_len = len(group)
-        if (group_len > _MAX_GROUP_NAME_LEN or group_len == 0):
-            raise ValueError('Group name len is out of range {_MAX_GROUP_NAME_LEN}')
-    else:
-        raise TypeError('Group must be a python str')
-
-    if isinstance(idx_list, list):
-        idx_len = len(idx_list)
-        if idx_len == 0:
-            raise ValueError('idx_list length is 0')
-    else:
-        raise TypeError('idx_list must be a python list')
-
-    for idx in idx_list:
-        if isinstance(idx, int):
-            if idx < 0:
-                raise ValueError('Idx < 0')
-        else:
-            raise TypeError('Idx in idx_list is invalid')
-
-    c_array_idx_list = _c_array(ctypes.c_uint, idx_list)
-    c_idx_num = ctypes.c_uint(len(idx_list))
-    c_group = _c_str(group)
-    ret = lib_ctype.HcomSetGradFusionByIndex(c_group, c_idx_num, c_array_idx_list)
-    if ret != 0:
-        raise RuntimeError('Allreduce split error')
-
-
-def _set_fusion_strategy_by_size(data_size_list, group="hccl_world_group"):
-    """
-    A function set gradient segment strategy according to the data size percentage list.
-
-    Note:
-        In the back propagation,
-        the fusion of the allreduce operators with a fusion attribute equals 1,
-        will be performed according to data_size_list,
-        to achieve the effect of parallel between calculation and communication.
-
-    Args:
-        data_size_list (list): The data size percentage list of the gradient.
-        group (str): The hccl communication group.
-
-    Raises:
-        TypeError: If group is not a python str.
-        TypeError: If data_size_list is not a python list.
-        TypeError: If type of data_size_list item is not int or float.
-        ValueError: If group name length is out of range.
-        ValueError: If data_size_list length is 0.
-        ValueError: If data_size_list item is less than 0.
-        RuntimeError: If allreduce split failed.
-    """
-    try:
-        lib_ctype = _load_lib()
-    except RuntimeError:
-        import hccl_test.manage.api as hccl
-        hccl.set_fusion_strategy_by_size()
-        return
-    finally:
-        pass
-
-    if isinstance(group, str):
-        group_len = len(group)
-        if group_len > _MAX_GROUP_NAME_LEN or group_len == 0:
-            raise ValueError('Group name is out of range {_MAX_GROUP_NAME_LEN}')
-    else:
-        raise TypeError('Group must be a python str')
-    if isinstance(data_size_list, list):
-        len_data_size = len(data_size_list)
-        if len_data_size == 0:
-            raise ValueError('data_size_list length is 0')
-    else:
-        raise TypeError('data_size_list must be a python list')
-    for data_size in data_size_list:
-        if not isinstance(data_size, (int, float)):
-            raise TypeError('data_size in data_size_list is invalid')
-
-    c_array_size_list = _c_array(ctypes.c_float, data_size_list)
-    c_size_num = ctypes.c_uint(len(data_size_list))
-    c_group = _c_str(group)
-    ret = lib_ctype.hcom_set_split_strategy_by_size(c_group, c_size_num, c_array_size_list)
-    if ret != 0:
-        raise RuntimeError('Allreduce split error')
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Data parallel allreduce fusion"""
+from __future__ import absolute_import
+from __future__ import division
+
+import ctypes
+
+_MAX_GROUP_NAME_LEN = 127
+_HCCL_LIB = 'libhccl.so'
+
+
+def _load_lib():
+    try:
+        hccl_lib = ctypes.CDLL(_HCCL_LIB)
+    except Exception:
+        raise RuntimeError('Get hccl lib error')
+
+    return hccl_lib
+
+
+def _c_str(string):
+    """Convert a python string to C string."""
+    if not isinstance(string, str):
+        string = string.decode('ascii')
+    return ctypes.c_char_p(string.encode('utf-8'))
+
+
+def _c_array(ctype, values):
+    """Create ctypes array from a python array."""
+    return (ctype * len(values))(*values)
+
+
+def _set_fusion_strategy_by_idx(idx_list, group="hccl_world_group"):
+    """
+    A function set gradient segment strategy according to the index list.
+
+    Note:
+        In the back propagation,
+        the fusion of the allreduce operators with a fusion attribute equals 1,
+        will be performed according to the idx_list,
+        to achieve the effect of parallel between calculation and communication.
+
+    Args:
+        idx_list (list): The index list of the gradient.
+        group (str): The hccl communication group.
+
+    Raises:
+        TypeError: If group is not a python str.
+        TypeError: If idx_list is not a python list.
+        TypeError: If type of idx_list item is not int.
+        ValueError: If group name length is out of range.
+        ValueError: If idx_list length is 0.
+        ValueError: If idx_list item is less than 0.
+        RuntimeError: If allreduce split failed.
+    """
+    try:
+        lib_ctype = _load_lib()
+    except RuntimeError:
+        import hccl_test.manage.api as hccl
+        hccl.set_fusion_strategy_by_idx()
+        return
+    finally:
+        pass
+    if isinstance(group, str):
+        group_len = len(group)
+        if (group_len > _MAX_GROUP_NAME_LEN or group_len == 0):
+            raise ValueError('Group name len is out of range {_MAX_GROUP_NAME_LEN}')
+    else:
+        raise TypeError('Group must be a python str')
+
+    if isinstance(idx_list, list):
+        idx_len = len(idx_list)
+        if idx_len == 0:
+            raise ValueError('idx_list length is 0')
+    else:
+        raise TypeError('idx_list must be a python list')
+
+    for idx in idx_list:
+        if isinstance(idx, int):
+            if idx < 0:
+                raise ValueError('Idx < 0')
+        else:
+            raise TypeError('Idx in idx_list is invalid')
+
+    c_array_idx_list = _c_array(ctypes.c_uint, idx_list)
+    c_idx_num = ctypes.c_uint(len(idx_list))
+    c_group = _c_str(group)
+    ret = lib_ctype.HcomSetGradFusionByIndex(c_group, c_idx_num, c_array_idx_list)
+    if ret != 0:
+        raise RuntimeError('Allreduce split error')
+
+
+def _set_fusion_strategy_by_size(data_size_list, group="hccl_world_group"):
+    """
+    A function set gradient segment strategy according to the data size percentage list.
+
+    Note:
+        In the back propagation,
+        the fusion of the allreduce operators with a fusion attribute equals 1,
+        will be performed according to data_size_list,
+        to achieve the effect of parallel between calculation and communication.
+
+    Args:
+        data_size_list (list): The data size percentage list of the gradient.
+        group (str): The hccl communication group.
+
+    Raises:
+        TypeError: If group is not a python str.
+        TypeError: If data_size_list is not a python list.
+        TypeError: If type of data_size_list item is not int or float.
+        ValueError: If group name length is out of range.
+        ValueError: If data_size_list length is 0.
+        ValueError: If data_size_list item is less than 0.
+        RuntimeError: If allreduce split failed.
+    """
+    try:
+        lib_ctype = _load_lib()
+    except RuntimeError:
+        import hccl_test.manage.api as hccl
+        hccl.set_fusion_strategy_by_size()
+        return
+    finally:
+        pass
+
+    if isinstance(group, str):
+        group_len = len(group)
+        if group_len > _MAX_GROUP_NAME_LEN or group_len == 0:
+            raise ValueError('Group name is out of range {_MAX_GROUP_NAME_LEN}')
+    else:
+        raise TypeError('Group must be a python str')
+    if isinstance(data_size_list, list):
+        len_data_size = len(data_size_list)
+        if len_data_size == 0:
+            raise ValueError('data_size_list length is 0')
+    else:
+        raise TypeError('data_size_list must be a python list')
+    for data_size in data_size_list:
+        if not isinstance(data_size, (int, float)):
+            raise TypeError('data_size in data_size_list is invalid')
+
+    c_array_size_list = _c_array(ctypes.c_float, data_size_list)
+    c_size_num = ctypes.c_uint(len(data_size_list))
+    c_group = _c_str(group)
+    ret = lib_ctype.hcom_set_split_strategy_by_size(c_group, c_size_num, c_array_size_list)
+    if ret != 0:
+        raise RuntimeError('Allreduce split error')

mindspore/parallel/_parallel_serialization.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Huawei Technologies Co., Ltd
+# Copyright 2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -28,7 +28,7 @@ from mindspore.parallel._tensor import _get_tensor_strategy, _construct_from_to_
 MAX_PATH_LENGTH = 1024
 
 
-def _convert_to_list(strategy):
+def _convert_to_list(strategy, rank_id=None):
     """Convert ParallelLayouts object to specified list."""
     train_map = {}
     for param_name in strategy.keys():
@@ -37,18 +37,34 @@ def _convert_to_list(strategy):
             dev_mat = list(layout.dev_matrix[0].dim)
             tensor_map = list(layout.tensor_map[0].dim)
             param_split_shape = list(layout.param_split_shape[0].dim)
-            field_size = int(layout.field)
-            shard_stride = int(layout.opt_weight_shard_step)
-            shard_size = int(layout.opt_weight_shard_size)
             pipeline_stage = 0
             origin_param_name = param_name
             if "-" in param_name:
                 pipeline_stage, origin_param_name = param_name.split("-")
+                pipeline_stage = int(pipeline_stage)
             if origin_param_name not in train_map:
-                train_map[origin_param_name] = [dev_mat, tensor_map, param_split_shape, field_size, shard_stride,
-                                                shard_size, [int(pipeline_stage)]]
+                train_map[origin_param_name] = [dev_mat, tensor_map, param_split_shape, int(layout.field),
+                                                int(layout.opt_weight_shard_step), int(layout.opt_weight_shard_size),
+                                                [pipeline_stage]]
             else:
-                train_map.get(origin_param_name)[6].append(int(pipeline_stage))
+                update_pipeline_stage_list = train_map.get(origin_param_name)[6] + [pipeline_stage]
+                if rank_id is not None:
+                    stage_device_num = np.prod(dev_mat)
+                    is_device0_and_pipeline0 = ((rank_id // stage_device_num) == 0) and (pipeline_stage == 0)
+                    not_device0_nor_pipeline0 = ((rank_id // stage_device_num) > 0) and (pipeline_stage > 0)
+                    if is_device0_and_pipeline0 or not_device0_nor_pipeline0:
+                        train_map[origin_param_name] = [dev_mat, tensor_map, param_split_shape,
+                                                        int(layout.field), int(layout.opt_weight_shard_step),
+                                                        int(layout.opt_weight_shard_size), update_pipeline_stage_list]
+                    else:
+                        train_map.get(origin_param_name)[6] = update_pipeline_stage_list
+                else:
+                    if np.all(pipeline_stage <= np.array(update_pipeline_stage_list)):
+                        train_map[origin_param_name] = [dev_mat, tensor_map, param_split_shape,
+                                                        int(layout.field), int(layout.opt_weight_shard_step),
+                                                        int(layout.opt_weight_shard_size), update_pipeline_stage_list]
+                    else:
+                        train_map.get(origin_param_name)[6] = update_pipeline_stage_list
         except BaseException as e:
             raise ValueError(f"{e.__str__()}. Convert layout strategy to list "
                              f"failed, please make sure that strategy matches the node_strategy.proto, you can "
@@ -219,12 +235,12 @@ def _parameter_not_in_local_stage(param_name, origin_strategy_list, strategy_lis
     return param_name in origin_strategy_list and param_name not in strategy_list
 
 
-def _extract_layout_map(strategy_file):
+def _extract_layout_map(strategy_file, rank_id=None):
     """Extract layout map"""
     layout_map = None
     if strategy_file is not None:
         src_strategy = _build_searched_strategy(strategy_file)
-        layout_map = _convert_to_list(src_strategy)
+        layout_map = _convert_to_list(src_strategy, rank_id)
     return layout_map
 
 
@@ -272,8 +288,8 @@ def _extract_src_dst_layout_map_by_src(src_strategy_file=None, dst_strategy_file
 
 def _extract_src_dst_layout_map(rank_id, src_strategy_file=None, dst_strategy_file=None):
     """Extract strategy list"""
-    src_layout_map = _extract_layout_map(src_strategy_file)
-    dst_layout_map = _extract_layout_map(dst_strategy_file)
+    src_layout_map = _extract_layout_map(src_strategy_file, None)
+    dst_layout_map = _extract_layout_map(dst_strategy_file, rank_id)
     if dst_layout_map is None:
         return src_layout_map, dst_layout_map
     dst_stage_device_num = np.prod(dst_layout_map.get(list(dst_layout_map.keys())[0])[0])
@@ -431,7 +447,7 @@ def _transform_parallel_checkpoint(rank_id, param_total_dict, param_attr_dict, s
         if param_type_dict[param_name][rank_id % device_num] == "BFloat16":
             transform_para.set_dtype(ms.bfloat16)
         transform_param_dict[param_name] = transform_para
-    if device_num < 0:
+    if device_num < 1:
         raise ValueError("None of the parameters in checkpoint file are in either src strategy or "
                          "dst strategy. Please check correctness of strategy files.")
 

mindspore/parallel/_ps_context.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Huawei Technologies Co., Ltd
+# Copyright 2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mindspore/parallel/_recovery_context.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Huawei Technologies Co., Ltd
+# Copyright 2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mindspore/parallel/_tensor.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Huawei Technologies Co., Ltd
+# Copyright 2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,8 +16,8 @@
 from __future__ import division
 from __future__ import absolute_import
 
+import copy
 import numpy as np
-from mindspore.common import dtype as mstype
 from mindspore.common.tensor import Tensor
 from mindspore.communication.management import get_rank, get_group_size
 from mindspore._c_expression import TensorTransform
@@ -198,7 +198,7 @@ def _get_slice_index(dev_mat, tensor_map, opt_shard_group):
     return tensor_slice_index
 
 
-def _load_tensor(tensor, dev_mat, tensor_map, rank_id=-1):
+def _load_tensor(tensor, dev_mat, tensor_map, full_shape=None, rank_id=-1):
     """
     Get the tensor slice of the local device by the device matrix and the tensor map
 
@@ -214,7 +214,8 @@ def _load_tensor(tensor, dev_mat, tensor_map, rank_id=-1):
         >>> tensor = Tensor(np.ones([32, 32]))
         >>> dev_mat = [2, 4]
         >>> tensor_map = [1, -1]
-        >>> tensor_slice = _load_tensor(tensor, dev_mat, tensor_map)
+        >>> full_shape = [32, 32]
+        >>> tensor_slice = _load_tensor(tensor, dev_mat, tensor_map, full_shape)
     """
     if rank_id == -1:
         rank = get_rank()
@@ -222,11 +223,9 @@ def _load_tensor(tensor, dev_mat, tensor_map, rank_id=-1):
         rank = rank_id
     tensor_strategy = _get_tensor_strategy(dev_mat, tensor_map)
     tensor_slice_index = _get_tensor_slice_index(dev_mat, tensor_strategy, tensor_map, rank)
-    if tensor.dtype == mstype.bfloat16:
-        from mindspore.ops.operations import Cast
-        cpu_cast = Cast().set_device("CPU")
-        tensor = cpu_cast(tensor, mstype.float32)
     np_tensor = tensor.asnumpy()
+    if full_shape:
+        np_tensor = np_tensor.reshape(full_shape)
     np_tensor_list = _chunk_tensor_by_strategy(np_tensor, tensor_strategy)
     np_tensor_slice = np_tensor_list[int(tensor_slice_index)]
     return np_tensor_slice
@@ -249,21 +248,29 @@ def _load_tensor_by_layout(tensor, layout, rank_id):
     """
     if not isinstance(layout, tuple):
         raise TypeError("The layout should be tuple! layout is {}".format(layout))
-    if len(layout) < 6:
-        raise ValueError("The length of layout must be larger than 5! layout is {}".format(layout))
+    if len(layout) < 7:
+        raise ValueError("The length of layout must be larger than 6! layout is {}".format(layout))
     dev_mat = layout[0]
     tensor_map = layout[1]
+    slice_shape = layout[2]
     if not tensor_map:
         return tensor
     uniform_split = layout[4]
     group = layout[5]
+    full_shape = layout[6]
     if uniform_split == 0:
         raise RuntimeError("The load tensor only support uniform split now")
-    tensor_slice = _load_tensor(tensor, dev_mat, tensor_map, rank_id)
+    tensor_slice = _load_tensor(tensor, dev_mat, tensor_map, full_shape, rank_id)
+    if tensor_slice.shape != slice_shape and not group:
+        tensor_slice = tensor_slice.reshape(slice_shape)
     if group:
         # get a totally shard tensor slice for parallel optimizer
         rank = get_rank(group)
         size = get_group_size(group)
+        if tensor_slice.shape != tuple(slice_shape) and slice_shape:
+            slice_shape_extend = copy.deepcopy(slice_shape)
+            slice_shape_extend[0] = slice_shape[0] * size
+            tensor_slice = tensor_slice.reshape(slice_shape_extend)
         tensor_slice = np.split(tensor_slice, size)[rank]
     return Tensor(tensor_slice, tensor.dtype)
 

mindspore/parallel/_transformer/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
+# Copyright 2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mindspore/parallel/_transformer/layers.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
+# Copyright 2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mindspore/parallel/_transformer/loss.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
+# Copyright 2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mindspore/parallel/_transformer/moe.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
+# Copyright 2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mindspore/parallel/_transformer/op_parallel_config.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
+# Copyright 2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mindspore/parallel/_transformer/transformer.py

@@ -1,4 +1,4 @@
-# Copyright 2021-2022 Huawei Technologies Co., Ltd
+# Copyright 2021-2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -400,7 +400,7 @@ class FeedForward(Cell):
             >>> from mindspore.nn.transformer import FeedForward
             >>> from mindspore import dtype as mstype
             >>> from mindspore import Tensor, nn
-            >>> import mindspore.ops as ops
+            >>> from mindspore import ops
             >>> model = FeedForward(hidden_size=15, ffn_hidden_size=30, dropout_rate=0.1)
             >>> tensor = Tensor(np.ones((2, 20, 15)), mstype.float32)
             >>> output = model(tensor)