mindspore 2.2.14__cp39-cp39-win_amd64.whl → 2.4.0__cp39-cp39-win_amd64.whl

mindspore/parallel/_dp_allreduce_fusion.py

@@ -1,159 +1,159 @@
-# Copyright 2020 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""Data parallel allreduce fusion"""
-from __future__ import absolute_import
-from __future__ import division
-
-import ctypes
-
-_MAX_GROUP_NAME_LEN = 127
-_HCCL_LIB = 'libhccl.so'
-
-
-def _load_lib():
-    try:
-        hccl_lib = ctypes.CDLL(_HCCL_LIB)
-    except Exception:
-        raise RuntimeError('Get hccl lib error')
-
-    return hccl_lib
-
-
-def _c_str(string):
-    """Convert a python string to C string."""
-    if not isinstance(string, str):
-        string = string.decode('ascii')
-    return ctypes.c_char_p(string.encode('utf-8'))
-
-
-def _c_array(ctype, values):
-    """Create ctypes array from a python array."""
-    return (ctype * len(values))(*values)
-
-
-def _set_fusion_strategy_by_idx(idx_list, group="hccl_world_group"):
-    """
-    A function set gradient segment strategy according to the index list.
-
-    Note:
-        In the back propagation,
-        the fusion of the allreduce operators with a fusion attribute equals 1,
-        will be performed according to the idx_list,
-        to achieve the effect of parallel between calculation and communication.
-
-    Args:
-        idx_list (list): The index list of the gradient.
-        group (str): The hccl communication group.
-
-    Raises:
-        TypeError: If group is not a python str.
-        TypeError: If idx_list is not a python list.
-        TypeError: If type of idx_list item is not int.
-        ValueError: If group name length is out of range.
-        ValueError: If idx_list length is 0.
-        ValueError: If idx_list item is less than 0.
-        RuntimeError: If allreduce split failed.
-    """
-    try:
-        lib_ctype = _load_lib()
-    except RuntimeError:
-        import hccl_test.manage.api as hccl
-        hccl.set_fusion_strategy_by_idx()
-        return
-    finally:
-        pass
-    if isinstance(group, str):
-        group_len = len(group)
-        if (group_len > _MAX_GROUP_NAME_LEN or group_len == 0):
-            raise ValueError('Group name len is out of range {_MAX_GROUP_NAME_LEN}')
-    else:
-        raise TypeError('Group must be a python str')
-
-    if isinstance(idx_list, list):
-        idx_len = len(idx_list)
-        if idx_len == 0:
-            raise ValueError('idx_list length is 0')
-    else:
-        raise TypeError('idx_list must be a python list')
-
-    for idx in idx_list:
-        if isinstance(idx, int):
-            if idx < 0:
-                raise ValueError('Idx < 0')
-        else:
-            raise TypeError('Idx in idx_list is invalid')
-
-    c_array_idx_list = _c_array(ctypes.c_uint, idx_list)
-    c_idx_num = ctypes.c_uint(len(idx_list))
-    c_group = _c_str(group)
-    ret = lib_ctype.HcomSetGradFusionByIndex(c_group, c_idx_num, c_array_idx_list)
-    if ret != 0:
-        raise RuntimeError('Allreduce split error')
-
-
-def _set_fusion_strategy_by_size(data_size_list, group="hccl_world_group"):
-    """
-    A function set gradient segment strategy according to the data size percentage list.
-
-    Note:
-        In the back propagation,
-        the fusion of the allreduce operators with a fusion attribute equals 1,
-        will be performed according to data_size_list,
-        to achieve the effect of parallel between calculation and communication.
-
-    Args:
-        data_size_list (list): The data size percentage list of the gradient.
-        group (str): The hccl communication group.
-
-    Raises:
-        TypeError: If group is not a python str.
-        TypeError: If data_size_list is not a python list.
-        TypeError: If type of data_size_list item is not int or float.
-        ValueError: If group name length is out of range.
-        ValueError: If data_size_list length is 0.
-        ValueError: If data_size_list item is less than 0.
-        RuntimeError: If allreduce split failed.
-    """
-    try:
-        lib_ctype = _load_lib()
-    except RuntimeError:
-        import hccl_test.manage.api as hccl
-        hccl.set_fusion_strategy_by_size()
-        return
-    finally:
-        pass
-
-    if isinstance(group, str):
-        group_len = len(group)
-        if group_len > _MAX_GROUP_NAME_LEN or group_len == 0:
-            raise ValueError('Group name is out of range {_MAX_GROUP_NAME_LEN}')
-    else:
-        raise TypeError('Group must be a python str')
-    if isinstance(data_size_list, list):
-        len_data_size = len(data_size_list)
-        if len_data_size == 0:
-            raise ValueError('data_size_list length is 0')
-    else:
-        raise TypeError('data_size_list must be a python list')
-    for data_size in data_size_list:
-        if not isinstance(data_size, (int, float)):
-            raise TypeError('data_size in data_size_list is invalid')
-
-    c_array_size_list = _c_array(ctypes.c_float, data_size_list)
-    c_size_num = ctypes.c_uint(len(data_size_list))
-    c_group = _c_str(group)
-    ret = lib_ctype.hcom_set_split_strategy_by_size(c_group, c_size_num, c_array_size_list)
-    if ret != 0:
-        raise RuntimeError('Allreduce split error')
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Data parallel allreduce fusion"""
+from __future__ import absolute_import
+from __future__ import division
+
+import ctypes
+
+_MAX_GROUP_NAME_LEN = 127
+_HCCL_LIB = 'libhccl.so'
+
+
+def _load_lib():
+    try:
+        hccl_lib = ctypes.CDLL(_HCCL_LIB)
+    except Exception:
+        raise RuntimeError('Get hccl lib error')
+
+    return hccl_lib
+
+
+def _c_str(string):
+    """Convert a python string to C string."""
+    if not isinstance(string, str):
+        string = string.decode('ascii')
+    return ctypes.c_char_p(string.encode('utf-8'))
+
+
+def _c_array(ctype, values):
+    """Create ctypes array from a python array."""
+    return (ctype * len(values))(*values)
+
+
+def _set_fusion_strategy_by_idx(idx_list, group="hccl_world_group"):
+    """
+    A function set gradient segment strategy according to the index list.
+
+    Note:
+        In the back propagation,
+        the fusion of the allreduce operators with a fusion attribute equals 1,
+        will be performed according to the idx_list,
+        to achieve the effect of parallel between calculation and communication.
+
+    Args:
+        idx_list (list): The index list of the gradient.
+        group (str): The hccl communication group.
+
+    Raises:
+        TypeError: If group is not a python str.
+        TypeError: If idx_list is not a python list.
+        TypeError: If type of idx_list item is not int.
+        ValueError: If group name length is out of range.
+        ValueError: If idx_list length is 0.
+        ValueError: If idx_list item is less than 0.
+        RuntimeError: If allreduce split failed.
+    """
+    try:
+        lib_ctype = _load_lib()
+    except RuntimeError:
+        import hccl_test.manage.api as hccl
+        hccl.set_fusion_strategy_by_idx()
+        return
+    finally:
+        pass
+    if isinstance(group, str):
+        group_len = len(group)
+        if (group_len > _MAX_GROUP_NAME_LEN or group_len == 0):
+            raise ValueError('Group name len is out of range {_MAX_GROUP_NAME_LEN}')
+    else:
+        raise TypeError('Group must be a python str')
+
+    if isinstance(idx_list, list):
+        idx_len = len(idx_list)
+        if idx_len == 0:
+            raise ValueError('idx_list length is 0')
+    else:
+        raise TypeError('idx_list must be a python list')
+
+    for idx in idx_list:
+        if isinstance(idx, int):
+            if idx < 0:
+                raise ValueError('Idx < 0')
+        else:
+            raise TypeError('Idx in idx_list is invalid')
+
+    c_array_idx_list = _c_array(ctypes.c_uint, idx_list)
+    c_idx_num = ctypes.c_uint(len(idx_list))
+    c_group = _c_str(group)
+    ret = lib_ctype.HcomSetGradFusionByIndex(c_group, c_idx_num, c_array_idx_list)
+    if ret != 0:
+        raise RuntimeError('Allreduce split error')
+
+
+def _set_fusion_strategy_by_size(data_size_list, group="hccl_world_group"):
+    """
+    A function set gradient segment strategy according to the data size percentage list.
+
+    Note:
+        In the back propagation,
+        the fusion of the allreduce operators with a fusion attribute equals 1,
+        will be performed according to data_size_list,
+        to achieve the effect of parallel between calculation and communication.
+
+    Args:
+        data_size_list (list): The data size percentage list of the gradient.
+        group (str): The hccl communication group.
+
+    Raises:
+        TypeError: If group is not a python str.
+        TypeError: If data_size_list is not a python list.
+        TypeError: If type of data_size_list item is not int or float.
+        ValueError: If group name length is out of range.
+        ValueError: If data_size_list length is 0.
+        ValueError: If data_size_list item is less than 0.
+        RuntimeError: If allreduce split failed.
+    """
+    try:
+        lib_ctype = _load_lib()
+    except RuntimeError:
+        import hccl_test.manage.api as hccl
+        hccl.set_fusion_strategy_by_size()
+        return
+    finally:
+        pass
+
+    if isinstance(group, str):
+        group_len = len(group)
+        if group_len > _MAX_GROUP_NAME_LEN or group_len == 0:
+            raise ValueError('Group name is out of range {_MAX_GROUP_NAME_LEN}')
+    else:
+        raise TypeError('Group must be a python str')
+    if isinstance(data_size_list, list):
+        len_data_size = len(data_size_list)
+        if len_data_size == 0:
+            raise ValueError('data_size_list length is 0')
+    else:
+        raise TypeError('data_size_list must be a python list')
+    for data_size in data_size_list:
+        if not isinstance(data_size, (int, float)):
+            raise TypeError('data_size in data_size_list is invalid')
+
+    c_array_size_list = _c_array(ctypes.c_float, data_size_list)
+    c_size_num = ctypes.c_uint(len(data_size_list))
+    c_group = _c_str(group)
+    ret = lib_ctype.hcom_set_split_strategy_by_size(c_group, c_size_num, c_array_size_list)
+    if ret != 0:
+        raise RuntimeError('Allreduce split error')

mindspore/parallel/_parallel_serialization.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Huawei Technologies Co., Ltd
+# Copyright 2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -24,11 +24,10 @@ from mindspore.parallel._tensor import _get_tensor_strategy, _construct_from_to_
     _generate_transform_operator_stack, _apply_tensor_transform_operators, _construct_tensor_layout_for_opt_shard, \
     _extract_layout_item
 
-
 MAX_PATH_LENGTH = 1024
 
 
-def _convert_to_list(strategy):
+def _convert_to_list(strategy, rank_id=None):
     """Convert ParallelLayouts object to specified list."""
     train_map = {}
     for param_name in strategy.keys():
@@ -44,11 +43,30 @@ def _convert_to_list(strategy):
             origin_param_name = param_name
             if "-" in param_name:
                 pipeline_stage, origin_param_name = param_name.split("-")
+                pipeline_stage = int(pipeline_stage)
             if origin_param_name not in train_map:
-                train_map[origin_param_name] = [dev_mat, tensor_map, param_split_shape, field_size, shard_stride,
-                                                shard_size, [int(pipeline_stage)]]
+                train_map[origin_param_name] = [dev_mat, tensor_map, param_split_shape, field_size,
+                                                shard_stride, shard_size,
+                                                [pipeline_stage]]
             else:
-                train_map.get(origin_param_name)[6].append(int(pipeline_stage))
+                update_pipeline_stage_list = train_map.get(origin_param_name)[6] + [pipeline_stage]
+                if rank_id is not None:
+                    stage_device_num = np.prod(dev_mat)
+                    is_device0_and_pipeline0 = ((rank_id // stage_device_num) == 0) and (pipeline_stage == 0)
+                    not_device0_nor_pipeline0 = ((rank_id // stage_device_num) > 0) and (pipeline_stage > 0)
+                    if is_device0_and_pipeline0 or not_device0_nor_pipeline0:
+                        train_map[origin_param_name] = [dev_mat, tensor_map, param_split_shape,
+                                                        field_size, shard_stride,
+                                                        shard_size, update_pipeline_stage_list]
+                    else:
+                        train_map.get(origin_param_name)[6] = update_pipeline_stage_list
+                else:
+                    if np.all(pipeline_stage <= np.array(update_pipeline_stage_list)):
+                        train_map[origin_param_name] = [dev_mat, tensor_map, param_split_shape,
+                                                        field_size, shard_stride,
+                                                        shard_size, update_pipeline_stage_list]
+                    else:
+                        train_map.get(origin_param_name)[6] = update_pipeline_stage_list
         except BaseException as e:
             raise ValueError(f"{e.__str__()}. Convert layout strategy to list "
                              f"failed, please make sure that strategy matches the node_strategy.proto, you can "
@@ -158,6 +176,8 @@ def _build_json_strategy(strategy_filename):
 
 def _build_searched_strategy(strategy_filename):
     """build searched strategy"""
+    if strategy_filename is None:
+        return strategy_filename
     _check_strategy_file(strategy_filename)
     if strategy_filename[-5:] != ".json":
         return _build_protobuf_strategy(strategy_filename)
@@ -219,12 +239,15 @@ def _parameter_not_in_local_stage(param_name, origin_strategy_list, strategy_lis
     return param_name in origin_strategy_list and param_name not in strategy_list
 
 
-def _extract_layout_map(strategy_file):
+def _extract_layout_map(strategy_file, rank_id=None):
     """Extract layout map"""
     layout_map = None
     if strategy_file is not None:
-        src_strategy = _build_searched_strategy(strategy_file)
-        layout_map = _convert_to_list(src_strategy)
+        if not isinstance(strategy_file, dict):
+            src_strategy = _build_searched_strategy(strategy_file)
+        else:
+            src_strategy = strategy_file
+        layout_map = _convert_to_list(src_strategy, rank_id)
     return layout_map
 
 
@@ -232,7 +255,10 @@ def _extract_pipeline_stage_num(strategy_file):
     """extract pipeline stage num"""
     pipeline_stage_num = 1
     if strategy_file is not None:
-        src_strategy = _build_searched_strategy(strategy_file)
+        if not isinstance(strategy_file, dict):
+            src_strategy = _build_searched_strategy(strategy_file)
+        else:
+            src_strategy = strategy_file
         layout_map = _convert_to_list(src_strategy)
         pipeline_stage_set = set()
         for _, layout in layout_map.items():
@@ -272,8 +298,8 @@ def _extract_src_dst_layout_map_by_src(src_strategy_file=None, dst_strategy_file
 
 def _extract_src_dst_layout_map(rank_id, src_strategy_file=None, dst_strategy_file=None):
     """Extract strategy list"""
-    src_layout_map = _extract_layout_map(src_strategy_file)
-    dst_layout_map = _extract_layout_map(dst_strategy_file)
+    src_layout_map = _extract_layout_map(src_strategy_file, None)
+    dst_layout_map = _extract_layout_map(dst_strategy_file, rank_id)
     if dst_layout_map is None:
         return src_layout_map, dst_layout_map
     dst_stage_device_num = np.prod(dst_layout_map.get(list(dst_layout_map.keys())[0])[0])
@@ -307,7 +333,10 @@ def _get_device_num_from_strategy(strategy_file=None):
     """Get device num from strategy file"""
     if strategy_file is None:
         return 1
-    src_strategy = _build_searched_strategy(strategy_file)
+    if not isinstance(strategy_file, dict):
+        src_strategy = _build_searched_strategy(strategy_file)
+    else:
+        src_strategy = strategy_file
     strategy_list = _convert_to_list(src_strategy)
     device_mat = list(strategy_list.values())[0][0]
     return np.prod(device_mat)
@@ -325,14 +354,15 @@ def _rank_list_for_transform_parallel_checkpoint(rank_id, src_strategy_list, dst
         from_dev_matrix, from_tensor_map, from_opt_shard_step, from_opt_shard_size = _extract_layout_item(
             src_strategy_list.get(param_name))
         from_device_num = np.prod(from_dev_matrix)
-        fake_tensor_shape = [8] * len(from_tensor_map)
         to_dev_matrix = [1]
-        to_tensor_map = [-1] * len(fake_tensor_shape)
+        to_tensor_map = [-1] * len(from_tensor_map)
         to_opt_shard_step = 0
         to_opt_shard_size = 0
         if dst_strategy_list is not None:
             to_dev_matrix, to_tensor_map, to_opt_shard_step, to_opt_shard_size = _extract_layout_item(
                 dst_strategy_list.get(param_name))
+        to_device_num = np.prod(to_dev_matrix)
+        fake_tensor_shape = [max(from_device_num, to_device_num)] * len(from_tensor_map)
         handled_key = (from_dev_matrix, from_tensor_map, from_opt_shard_step, from_opt_shard_size,
                        to_dev_matrix, to_tensor_map, to_opt_shard_step, to_opt_shard_size)
         if handled_key in handled_layout:
@@ -417,7 +447,6 @@ def _transform_parallel_checkpoint(rank_id, param_total_dict, param_attr_dict, s
         param_rank_map = _get_needed_rank_transform_operator_map_by_layouts(from_tensor_layout, to_tensor_layout,
                                                                             device_list, rank_id)
 
-
         from_info_tuple = (from_opt_shard_size, from_dev_matrix, from_tensor_map, from_full_tensor_shape)
         to_info_tuple = (to_opt_shard_size, to_dev_matrix_origin, to_tensor_map_origin, origin_tensor_shape)
         _insert_opt_shard_reshape(param_rank_map, from_info_tuple, to_info_tuple)
@@ -427,24 +456,24 @@ def _transform_parallel_checkpoint(rank_id, param_total_dict, param_attr_dict, s
         transform_tensor = ms.Tensor(param_total_dict_copy[rank_id % device_num])
         requires_grad = param_attr_dict[param_name][rank_id % device_num][0]
         layerwise_parallel = param_attr_dict[param_name][rank_id % device_num][1]
-        transform_para = ms.Parameter(transform_tensor, param_name, requires_grad, layerwise_parallel)
+        transform_param = ms.Parameter(transform_tensor, param_name, requires_grad, layerwise_parallel)
         if param_type_dict[param_name][rank_id % device_num] == "BFloat16":
-            transform_para.set_dtype(ms.bfloat16)
-        transform_param_dict[param_name] = transform_para
-    if device_num < 0:
+            transform_param.set_dtype(ms.bfloat16)
+        transform_param_dict[param_name] = transform_param
+    if device_num < 1:
         raise ValueError("None of the parameters in checkpoint file are in either src strategy or "
                          "dst strategy. Please check correctness of strategy files.")
 
     # Handle those parameter like learning_rate, global_step which not in strategy_file.
     for param_name, _ in param_total_dict.items():
         if param_name not in transform_param_dict:
-            transform_para = ms.Parameter(
+            transform_param = ms.Parameter(
                 ms.Tensor(param_total_dict[param_name][rank_id % device_num]), param_name,
                 param_attr_dict[param_name][rank_id % device_num][0],
                 param_attr_dict[param_name][rank_id % device_num][1])
             if param_type_dict[param_name][rank_id % device_num] == "BFloat16":
-                transform_para.set_dtype(ms.bfloat16)
-            transform_param_dict[param_name] = transform_para
+                transform_param.set_dtype(ms.bfloat16)
+            transform_param_dict[param_name] = transform_param
 
     transform_param_list = [{"name": param_name, "data": param_data}
                             for param_name, param_data in transform_param_dict.items()]
@@ -515,3 +544,18 @@ def _insert_opt_shard_reshape(param_rank_map, from_info_tuple, to_info_tuple):
                     continue
                 to_slice_tensor_shape += (item // to_tensor_strategy[i],)
             param_rank_map.get(param_rank).append(('Reshape', list(to_slice_tensor_shape)))
+
+
+def _get_param_list_when_first_dim_sharded(device_arrangement, first_dim_sharded_device_index, rank):
+    """Calculate rank list for optimizer parallel when first dim of parameter is sharded by other parallel method"""
+    total_device_num = 1
+    for n in device_arrangement:
+        total_device_num *= n
+    if first_dim_sharded_device_index != len(device_arrangement) - 1:
+        return list(range(0, total_device_num))
+    first_dim_sharded_size = device_arrangement[-1 - first_dim_sharded_device_index]
+    range_size = total_device_num // first_dim_sharded_size
+    offset = rank % range_size
+    start = rank - offset
+    param_total_list = list(range(start, start + range_size))
+    return param_total_list

mindspore/parallel/_ps_context.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Huawei Technologies Co., Ltd
+# Copyright 2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mindspore/parallel/_recovery_context.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Huawei Technologies Co., Ltd
+# Copyright 2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.