mindspore 2.2.14__cp39-cp39-win_amd64.whl → 2.4.0__cp39-cp39-win_amd64.whl

mindspore/parallel/_auto_parallel_context.py

@@ -1,4 +1,4 @@
-# Copyright 2020-2022 Huawei Technologies Co., Ltd
+# Copyright 2020-2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -168,6 +168,24 @@ class _AutoParallelContext:
         self.check_context_handle()
         return _ParallelFusionConfig.CONFIG
 
+    def set_dump_local_norm(self, dump_local_norm):
+        """
+        Set dump local norm for auto parallel.
+
+        Args:
+            dump_local_norm (bool): User need to specify if he want to dump local norm.  Default: False
+
+        Raises:
+            KeyError: When key of comm_fusion is not 'allreduce'.
+        """
+        self.check_context_handle()
+        self._context_handle.set_dump_local_norm(dump_local_norm)
+
+    def get_dump_local_norm(self):
+        """Get dump local norm."""
+        self.check_context_handle()
+        return self._context_handle.get_dump_local_norm()
+
     def set_fusion_threshold_mb(self, fusion_threshold=64, comm_type="allreduce"):
         """
         Set fusion threshold (MB) for auto parallel.
@@ -242,6 +260,38 @@ class _AutoParallelContext:
         self.check_context_handle()
         return self._context_handle.get_pipeline_stage_split_num()
 
+    def set_auto_pipeline(self, auto_pipeline):
+        """Set the pipeline stage number to automatic"""
+        if not isinstance(auto_pipeline, bool):
+            raise TypeError("For 'set_auto_parallel_context', the argument 'auto_pipeline' "
+                            "must be bool, but got the type : {}.".format(type(auto_pipeline)))
+        self.check_context_handle()
+        self._context_handle.set_auto_pipeline(auto_pipeline)
+
+    def get_auto_pipeline(self):
+        """Get whether the pipeline stage number is automatic"""
+        self.check_context_handle()
+        return self._context_handle.get_auto_pipeline()
+
+    def set_pipeline_result_broadcast(self, pipeline_result_broadcast):
+        """
+        Set the value of enabling pipeline result broadcast. Default: ``False``.
+
+        Args:
+            pipeline_result_broadcast (bool): Enable/disable broadcast the last stage result to all other stages.
+        """
+        self.check_context_handle()
+        if not isinstance(pipeline_result_broadcast, bool):
+            raise TypeError("For 'set_auto_parallel_context().set_pipeline_result_broadcast', the argument "
+                            "'pipeline_result_broadcast' must be bool, but got the type : {}."
+                            .format(type(pipeline_result_broadcast)))
+        self._context_handle.set_pipeline_result_broadcast(pipeline_result_broadcast)
+
+    def get_pipeline_result_broadcast(self):
+        """Get the value of enabling pipeline result broadcast"""
+        self.check_context_handle()
+        return self._context_handle.get_pipeline_result_broadcast()
+
     def get_pipeline_interleave(self):
         """Get pipeline interleave flag"""
         self.check_context_handle()
@@ -552,7 +602,7 @@ class _AutoParallelContext:
         self.check_context_handle()
         dir_path = os.path.dirname(strategy_ckpt_save_file)
         if dir_path and not os.path.exists(dir_path):
-            os.makedirs(dir_path)
+            os.makedirs(dir_path, mode=0o700, exist_ok=True)
         self._context_handle.set_strategy_ckpt_save_file(strategy_ckpt_save_file)
 
     def get_strategy_ckpt_save_file(self):
@@ -611,7 +661,7 @@ class _AutoParallelContext:
         self.check_context_handle()
         dir_path = os.path.dirname(group_ckpt_save_file)
         if dir_path and not os.path.exists(dir_path):
-            os.makedirs(dir_path)
+            os.makedirs(dir_path, mode=0o700, exist_ok=True)
         self._context_handle.set_group_ckpt_save_file(group_ckpt_save_file)
 
     def get_parameter_broadcast_is_set(self):
@@ -800,6 +850,20 @@ class _AutoParallelContext:
                             .format(type(enable_parallel_optimizer)))
         self._context_handle.set_enable_parallel_optimizer(enable_parallel_optimizer)
 
+    def set_force_fp32_communication(self, force_fp32_communication):
+        """
+        Set enable/disable force fp32 communication.
+
+        Args:
+            set_force_fp32_communication (bool): Enable/disable force fp32 communication.
+        """
+        self.check_context_handle()
+        if not isinstance(force_fp32_communication, bool):
+            raise TypeError("For 'set_auto_parallel_context', "
+                            "the argument 'force_fp32_communication' must be bool, but got the type : {}."
+                            .format(type(force_fp32_communication)))
+        self._context_handle.set_force_fp32_communication(force_fp32_communication)
+
     def get_enable_fold_pipeline(self):
         """Get parallel optimizer flag."""
         self.check_context_handle()
@@ -861,6 +925,12 @@ class _AutoParallelContext:
         self.check_context_handle()
         return self._context_handle.get_enable_parallel_optimizer()
 
+    def get_force_fp32_communication(self):
+        """Get force fp32 communication flag."""
+        self.check_context_handle()
+        return self._context_handle.get_force_fp32_communication()
+
+
     def set_parallel_optimizer_config(self, parallel_optimizer_config):
         r"""
         Set the configure for parallel optimizer. The configure provides more detailed behavior control about parallel
@@ -1065,9 +1135,9 @@ class _AutoParallelContext:
         """
         self.check_context_handle()
         if comm_type == "allgather" and not self.get_enable_all_gather_fusion():
-            return
+            self.set_enable_all_gather_fusion(True)
         if comm_type == "reducescatter" and not self.get_enable_reduce_scatter_fusion():
-            return
+            self.set_enable_reduce_scatter_fusion(True)
         if not isinstance(comm_fusion, dict):
             raise TypeError("For 'comm_fusion', {} config must be dict, but got the type : {}.".format(
                 comm_type, type(comm_fusion)))
@@ -1101,7 +1171,7 @@ class _AutoParallelContext:
         """
         self.check_context_handle()
         if not self.get_enable_all_reduce_fusion():
-            return
+            self.set_enable_all_reduce_fusion(True)
         if not isinstance(comm_fusion, dict):
             raise TypeError("For 'comm_fusion', the 'allreduce' config must be dict, but got the type : {}.".format(
                 type(comm_fusion)))
@@ -1158,7 +1228,7 @@ def _set_ops_strategy_json_config(type="SAVE", path="", mode="all"):
     """
     dir_path = os.path.dirname(path)
     if dir_path and not os.path.exists(dir_path):
-        os.makedirs(dir_path)
+        os.makedirs(dir_path, mode=0o700, exist_ok=True)
     check_type = ["SAVE", "LOAD"]
     check_mode = ["all", "principal"]
     if type in check_type and mode in check_mode:
@@ -1190,6 +1260,8 @@ _set_auto_parallel_context_func_map = {
     "gradient_fp32_sync": auto_parallel_context().set_gradient_fp32_sync,
     "loss_repeated_mean": auto_parallel_context().set_loss_repeated_mean,
     "pipeline_stages": auto_parallel_context().set_pipeline_stages,
+    "auto_pipeline": auto_parallel_context().set_auto_pipeline,
+    "pipeline_result_broadcast": auto_parallel_context().set_pipeline_result_broadcast,
     "pipeline_segments": auto_parallel_context().set_pipeline_segments,
     "parallel_mode": auto_parallel_context().set_parallel_mode,
     "search_mode": auto_parallel_context().set_strategy_search_mode,
@@ -1201,6 +1273,7 @@ _set_auto_parallel_context_func_map = {
     "full_batch": auto_parallel_context().set_full_batch,
     "dataset_strategy": auto_parallel_context().set_dataset_strategy,
     "enable_parallel_optimizer": auto_parallel_context().set_enable_parallel_optimizer,
+    "force_fp32_communication": auto_parallel_context().set_force_fp32_communication,
     "parallel_optimizer_config": auto_parallel_context().set_parallel_optimizer_config,
     "pipeline_config": auto_parallel_context().set_pipeline_config,
     "grad_accumulation_step": auto_parallel_context().set_grad_accumulation_step,
@@ -1211,7 +1284,8 @@ _set_auto_parallel_context_func_map = {
     "sharding_propagation": auto_parallel_context().set_sharding_propagation,
     "enable_alltoall": auto_parallel_context().set_enable_alltoall,
     "strategy_ckpt_config": auto_parallel_context().set_strategy_ckpt_config,
-    "comm_fusion": auto_parallel_context().set_comm_fusion}
+    "comm_fusion": auto_parallel_context().set_comm_fusion,
+    "dump_local_norm": auto_parallel_context().set_dump_local_norm}
 
 _get_auto_parallel_context_func_map = {
     "device_num": auto_parallel_context().get_device_num,
@@ -1220,6 +1294,8 @@ _get_auto_parallel_context_func_map = {
     "gradient_fp32_sync": auto_parallel_context().get_gradient_fp32_sync,
     "loss_repeated_mean": auto_parallel_context().get_loss_repeated_mean,
     "pipeline_stages": auto_parallel_context().get_pipeline_stages,
+    "auto_pipeline": auto_parallel_context().get_auto_pipeline,
+    "pipeline_result_broadcast": auto_parallel_context().get_pipeline_result_broadcast,
     "pipeline_interleave": auto_parallel_context().get_pipeline_interleave,
     "pipeline_scheduler": auto_parallel_context().get_pipeline_scheduler,
     "parallel_mode": auto_parallel_context().get_parallel_mode,
@@ -1231,6 +1307,7 @@ _get_auto_parallel_context_func_map = {
     "full_batch": auto_parallel_context().get_full_batch,
     "dataset_strategy": auto_parallel_context().get_dataset_strategy,
     "enable_parallel_optimizer": auto_parallel_context().get_enable_parallel_optimizer,
+    "force_fp32_communication": auto_parallel_context().get_force_fp32_communication,
     "grad_accumulation_step": auto_parallel_context().get_grad_accumulation_step,
     "all_reduce_fusion_config": auto_parallel_context().get_all_reduce_fusion_split_indices,
     "communi_parallel_mode": auto_parallel_context().get_communi_parallel_mode,
@@ -1240,7 +1317,8 @@ _get_auto_parallel_context_func_map = {
     "enable_alltoall": auto_parallel_context().get_enable_alltoall,
     "comm_fusion": auto_parallel_context().get_comm_fusion,
     "strategy_ckpt_config": auto_parallel_context().get_strategy_ckpt_config,
-    "full_batch_is_set": auto_parallel_context().get_full_batch_is_set}
+    "full_batch_is_set": auto_parallel_context().get_full_batch_is_set,
+    "dump_local_norm": auto_parallel_context().get_dump_local_norm}
 
 
 @args_type_check(device_num=int, global_rank=int, gradients_mean=bool, gradient_fp32_sync=bool,
@@ -1250,7 +1328,7 @@ _get_auto_parallel_context_func_map = {
                  grad_accumulation_step=int, all_reduce_fusion_config=list, group_ckpt_save_file=str,
                  communi_parallel_mode=str, optimizer_weight_shard_size=int, sharding_propagation=bool,
                  optimizer_weight_shard_aggregated_save=bool, enable_alltoall=bool, comm_fusion=dict,
-                 strategy_ckpt_config=dict)
+                 strategy_ckpt_config=dict, force_fp32_communication=bool)
 def _set_auto_parallel_context(**kwargs):
     """
     Set auto parallel context.
@@ -1298,11 +1376,19 @@ def _set_auto_parallel_context(**kwargs):
         full_batch (bool): Whether to load the whole batch on each device. Default: ``False``.
         dataset_strategy Union[str, tuple]: Dataset sharding strategy. Default: "data_parallel".
         enable_parallel_optimizer (bool): Enable using optimizer segmentation or not. Default: ``False``.
+        force_fp32_communication (bool): A switch that determines whether reduce operators (AllReduce, ReduceScatter)
+                        are forced to use the fp32 data type for communication during communication. True is the enable
+                        switch. Default: ``False`` .
         all_reduce_fusion_config (list): Set allreduce fusion strategy by parameters indices.
         pipeline_stages (int): Set the stage information for pipeline parallel. This indicates how
                         the devices are distributed alone the pipeline. The total devices will be divided into
                         'pipeline_stags' stages. This currently could only be used when
                         parallel mode semi_auto_parallel is enabled. Default: 0
+        auto_pipeline (bool): Set the pipeline stage number to automatic. Its value will be selected between 1 and the
+                        parameter `pipeline_stages`. This option requires the `parallel_mode` to be ``auto_parallel``
+                        and the `search_mode` to be ``recursive_programming``. Default: ``False`` .
+        pipeline_result_broadcast (bool): A switch that broadcast the last stage result to all other stage in pipeline
+                        parallel inference. Default: ``False`` .
         communi_parallel_mode (str): There are tree kinds of communication parallel modes, "all_group_parallel",
                      "same_server_group_parallel" and "no_group_parallel". Default: "all_group_parallel".
 
@@ -1344,6 +1430,7 @@ def _set_auto_parallel_context(**kwargs):
                         and `size`. Config is same as `allgather`.
 
 
+
     Raises:
         ValueError: If input key is not attribute in auto parallel context.
     """
@@ -1386,10 +1473,13 @@ def _reset_auto_parallel_context():
     - strategy_ckpt_load_file: ""
     - strategy_ckpt_save_file: ""
     - enable_parallel_optimizer: False
+    - force_fp32_communication: False
     - search_mode: 'recursive_programming
     - auto_parallel_search_mode: 'recursive_programming
     - sharding_propagation: False
     - pipeline_stages: 0
+    - auto_pipeline: False
+    - pipeline_result_broadcast: False
     - gradient_accumulation_shard: True
     - fusion_threshold: 64
     """

mindspore/parallel/_cell_wrapper.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Huawei Technologies Co., Ltd
+# Copyright 2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,10 +16,16 @@
 from __future__ import absolute_import
 from __future__ import division
 
+import numpy as np
+
+from mindspore import context
 from mindspore.nn.cell import Cell
+from mindspore.ops import operations as P
 from mindspore.ops.operations.comm_ops import AllGather
 from mindspore.communication import GlobalComm
 from mindspore.common import jit
+from mindspore.communication import create_group
+from mindspore.train._utils import get_parameter_redundancy, remove_param_redundancy
 
 _ALLGATHER_CELL = None
 
@@ -29,16 +35,19 @@ class AllGatherCell(Cell):
     Allgather cell, used in model parallel scenario.
     To allgather the selected parameter slice from each device.
     """
-    def __init__(self, group):
-        super(AllGatherCell, self).__init__(auto_prefix=False)
 
+    def __init__(self, group, do_reshape, after_reshape_slice_shape):
+        super(AllGatherCell, self).__init__(auto_prefix=False)
         self.allgather = AllGather(group)
+        self.do_reshape = do_reshape
+        self.after_reshape_slice_shape = tuple(after_reshape_slice_shape)
         self.add_flags(skip_auto_parallel_compile=True)
 
     @jit()
     def construct(self, x):
+        if self.do_reshape:
+            x = P.Reshape()(x, self.after_reshape_slice_shape)
         x = self.allgather(x)
-
         return x
 
 
@@ -51,29 +60,49 @@ class SaveOptShardCkptCell(Cell):
     Note:
         This could be optimized later with less communication consumption.
     """
-    def __init__(self, group):
+
+    def __init__(self, group, do_reshape, after_reshape_slice_shape):
         super(SaveOptShardCkptCell, self).__init__(auto_prefix=False)
         self.allgather1 = AllGather(group)
         self.allgather2 = AllGather()
+        self.do_reshape = do_reshape
+        self.after_reshape_slice_shape = tuple(after_reshape_slice_shape)
         self.add_flags(skip_auto_parallel_compile=True)
 
     def construct(self, x):
         x = self.allgather1(x)
+        if self.do_reshape:
+            x = P.Reshape()(x, self.after_reshape_slice_shape)
         x = self.allgather2(x)
 
         return x
 
 
-def get_allgather_cell(group, need_merge_twice=False):
+class SingleCommunicator(Cell):
+    """
+    Used to broadcast single parameter.
+    """
+
+    def __init__(self, group_name):
+        super(SingleCommunicator, self).__init__()
+        self.allreduce = P.AllReduce(group=group_name)
+        self.add_flags(skip_auto_parallel_compile=True)
+
+    def construct(self, loaded_param):
+        result = self.allreduce(loaded_param)
+        return result
+
+
+def get_allgather_cell(group, need_merge_twice=False, do_reshape=False, after_reshape_slice_shape=()):
     """Get AllGatherCell object."""
     global _ALLGATHER_CELL
     if need_merge_twice:
-        _ALLGATHER_CELL = SaveOptShardCkptCell(group)
+        _ALLGATHER_CELL = SaveOptShardCkptCell(group, do_reshape, after_reshape_slice_shape)
     else:
         if group:
-            _ALLGATHER_CELL = AllGatherCell(group)
+            _ALLGATHER_CELL = AllGatherCell(group, do_reshape, after_reshape_slice_shape)
         else:
-            _ALLGATHER_CELL = AllGatherCell(GlobalComm.WORLD_COMM_GROUP)
+            _ALLGATHER_CELL = AllGatherCell(GlobalComm.WORLD_COMM_GROUP, do_reshape, after_reshape_slice_shape)
     return _ALLGATHER_CELL
 
 
@@ -82,3 +111,64 @@ def destroy_allgather_cell():
     global _ALLGATHER_CELL
     if _ALLGATHER_CELL:
         _ALLGATHER_CELL = None
+
+
+def _chang_parallel_context(origin_dataset_strategy):
+    """Change the original parallel state."""
+    if context.get_context("mode") == context.GRAPH_MODE:
+        context.set_auto_parallel_context(parallel_mode="hybrid_parallel")
+        if origin_dataset_strategy != "data_parallel":
+            context.set_auto_parallel_context(dataset_strategy="data_parallel")
+
+
+def _restore_parallel_context(origin_parallel_mode, origin_dataset_strategy):
+    """Restore the original parallel state."""
+    if context.get_context("mode") == context.GRAPH_MODE:
+        context.set_auto_parallel_context(parallel_mode=origin_parallel_mode)
+        if origin_dataset_strategy != "data_parallel":
+            context.set_auto_parallel_context(dataset_strategy=origin_dataset_strategy)
+
+
+def _single_parameter_broadcast(net, layout, cur_rank=0, initial_rank=0):
+    """
+    Broadcast single parameter to other rank in data parallel dimension.
+    """
+    from mindspore import Tensor
+    origin_parallel_mode = context.get_auto_parallel_context("parallel_mode")
+    origin_dataset_strategy = context.get_auto_parallel_context("dataset_strategy")
+    if layout:
+        param_redundancy = get_parameter_redundancy(layout, initial_rank)
+    else:
+        param_redundancy = get_parameter_redundancy(net)
+    if not param_redundancy:
+        return
+    single_params = remove_param_redundancy(param_redundancy)
+    if not single_params:
+        return
+    param_redundancy_reversed = {}
+    for key, redundancy in param_redundancy.items():
+        for item in redundancy:
+            if len(item) == 1:
+                continue
+            if cur_rank in item:
+                param_redundancy_reversed.setdefault(item, []).append(key)
+    if not param_redundancy_reversed or cur_rank not in single_params:
+        return
+    net_param_dict = net.parameters_dict()
+    _chang_parallel_context(origin_dataset_strategy)
+    for group, params in param_redundancy_reversed.items():
+        create_group(str(group), list(group))
+        allreduce_input = []
+        for param in params:
+            if param not in net_param_dict:
+                continue
+            real_param = net_param_dict[param]
+            if param not in single_params[cur_rank]:
+                real_param.set_data(Tensor(np.zeros(real_param.shape), dtype=real_param.dtype), real_param.sliced)
+            allreduce_input.append(real_param)
+        if not allreduce_input:
+            continue
+        communicator = SingleCommunicator(str(group))
+        for real_param in allreduce_input:
+            real_param.set_data(communicator(real_param), real_param.sliced)
+    _restore_parallel_context(origin_parallel_mode, origin_dataset_strategy)

mindspore/parallel/_cost_model_context.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Huawei Technologies Co., Ltd
+# Copyright 2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.