mindspore 2.2.11__cp37-none-any.whl → 2.2.14__cp37-none-any.whl

mindspore/ops/silent_check.py

@@ -0,0 +1,162 @@
+# Copyright 2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Silent Check."""
+import os
+
+from mindspore.common.tensor import Tensor
+from mindspore.common.parameter import Parameter
+import mindspore.common.dtype as mstype
+
+from . import operations
+from .operations._inner_ops import _MirrorSilentCheck
+from .operations import RmsNorm as OriginRmsNorm
+from .operations import LayerNorm as OriginLayerNorm
+from .primitive import Primitive
+
+
+NPU_ASD_ENABLE = 'NPU_ASD_ENABLE'
+
+
+class ASDBase:
+    """
+    ASDBase is the base class of operator with accuracy-sensitive detection feature in python.
+
+    Args:
+        cls (Primitive): Original operator requiring accuracy-sensitive detection feature.
+        args (tuple):  A variable parameter tuple to the original operator.
+        kwargs (dict): A variable parameter dictionary passed the original operator.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> from mindspore.ops.silent_check import ASDBase
+        >>> from mindspore.ops import LayerNorm as OriginLayerNorm
+        >>> class LayerNormASD(ASDBase):
+        ...     def __init__(self, *args, **kwargs):
+        ...         super().__init__(OriginLayerNorm, *args, **kwargs)
+        ...         # init parameters for accuracy-sensitive detection by calling the base class method generate_params()
+        ...         self.pre_val, self.min_val, self.max_val, self.cnt = self.generate_params()
+        ...
+        ...     def __call__(self, input_x, gamma, beta):
+        ...         if self.enable_check:
+        ...             # execute accuracy-sensitive detection by calling the check_op of base class
+        ...             input_x = self.check_op(
+        ...                 input_x, self.pre_val, self.min_val, self.max_val, self.cnt, None)
+        ...             self.cnt += 1
+        ...         # return the result of original operator
+        ...         return self.op(input_x, gamma, beta)
+    """
+    _index = 0
+    __ms_class__ = True
+
+    def __init__(self, cls, *args, **kwargs):
+        self.op = cls(*args, **kwargs)
+        self.check_op = _MirrorSilentCheck()
+        self._suffix = "ASD_" + cls.__name__
+        primitive_attr = dir(Primitive)
+        self._op_attr_dict = {
+            name for name in primitive_attr if not name.startswith("_")}
+        self.enable_check = os.environ.get(NPU_ASD_ENABLE) == "1"
+
+    def __getattr__(self, name):
+        def method_wrapper(*args, **kwargs):
+            out = getattr(self.op, name)(*args, **kwargs)
+            if out is self.op:
+                return self
+            return out
+
+        if name in self._op_attr_dict:
+            if callable(getattr(self.op, name)):
+                return method_wrapper
+        if hasattr(self.op, name):
+            return getattr(self.op, name)
+        return super().__getattr__(self, name)
+
+    def __repr__(self):
+        return self.op.__repr__()
+
+    def generate_params(self):
+        """
+        Generate support params for accuracy-sensitive detection.
+
+        Returns:
+            tuple consisting of four elements.
+            The derived class initializes the parameters required for accuracy-sensitive detection by calling
+            this function.
+
+        Examples:
+            >>> from mindspore.ops.silent_check import ASDBase
+            >>> from mindspore.ops import LayerNorm as OriginLayerNorm
+            >>> class LayerNormASD(ASDBase):
+            ...     def __init__(self, *args, **kwargs):
+            ...         super().__init__(OriginLayerNorm, *args, **kwargs)
+            ...         # init parameters for accuracy-sensitive detection by calling the base class function
+            ...         self.pre_val, self.min_val, self.max_val, self.cnt = self.generate_params()
+        """
+        pre_val = Parameter(Tensor(0, mstype.float32),
+                            name=f"{self._suffix}_pre_val_{self._index}",
+                            requires_grad=False)
+        min_val = Parameter(Tensor(0, mstype.float32),
+                            name=f"{self._suffix}_min_val_{self._index}",
+                            requires_grad=False)
+        max_val = Parameter(Tensor(0, mstype.float32),
+                            name=f"{self._suffix}_max_val_{self._index}",
+                            requires_grad=False)
+        cnt = Parameter(Tensor(0, mstype.int32),
+                        name=f"{self._suffix}_cnt_{self._index}",
+                        requires_grad=False)
+        ASDBase._index += 1
+        return pre_val, min_val, max_val, cnt
+
+
+class RmsNormASD(ASDBase):
+    """
+    RmsNorm with ASD.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(OriginRmsNorm, *args, **kwargs)
+        self.pre_val, self.min_val, self.max_val, self.cnt = self.generate_params()
+
+    def __call__(self, input_x, gamma):
+        if self.enable_check:
+            input_x = self.check_op(
+                input_x, self.pre_val, self.min_val, self.max_val, self.cnt, None)
+            self.cnt += 1
+        return self.op(input_x, gamma)
+
+
+class LayerNormASD(ASDBase):
+    """
+    LayerNorm with ASD.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(OriginLayerNorm, *args, **kwargs)
+        self.pre_val, self.min_val, self.max_val, self.cnt = self.generate_params()
+
+    def __call__(self, input_x, gamma, beta):
+        if self.enable_check:
+            input_x = self.check_op(
+                input_x, self.pre_val, self.min_val, self.max_val, self.cnt, None)
+            self.cnt += 1
+        return self.op(input_x, gamma, beta)
+
+
+def _silent_check():
+    if os.environ.get(NPU_ASD_ENABLE) == "1":
+        operations.LayerNorm = LayerNormASD
+        operations.RmsNorm = RmsNormASD

mindspore/parallel/__init__.py

@@ -18,8 +18,9 @@ from __future__ import absolute_import
 from mindspore.parallel.algo_parameter_config import get_algo_parameters, reset_algo_parameters, \
     set_algo_parameters
 from mindspore.parallel.checkpoint_transform import rank_list_for_transform, transform_checkpoint_by_rank, \
-    transform_checkpoints, merge_pipeline_strategys
+    transform_checkpoints, merge_pipeline_strategys, load_segmented_checkpoints
 from mindspore.parallel.shard import shard
 
 __all__ = ["set_algo_parameters", "reset_algo_parameters", "get_algo_parameters", "rank_list_for_transform",
-           "transform_checkpoint_by_rank", "transform_checkpoints", "merge_pipeline_strategys", "shard"]
+           "transform_checkpoint_by_rank", "transform_checkpoints", "merge_pipeline_strategys", "shard",
+           "load_segmented_checkpoints"]

mindspore/parallel/_auto_parallel_context.py

@@ -65,6 +65,19 @@ class _ParallelOptimizerConfig:
     OPTIMIZER_WEIGHT_SHARD_SIZE = "optimizer_weight_shard_size"
 
 
+class _PipelineConfig:
+    """
+    The key of the Pipeline parallelism.
+    """
+    PIPELINE_INTERLEAVE = "pipeline_interleave"
+    PIPELINE_SCHEDULER = "pipeline_scheduler"
+
+
+class _PipelineScheduler:
+    PIPELINE_1F1B = "1f1b"
+    PIPELINE_GPIPE = "gpipe"
+
+
 class _AutoParallelContext:
     """
     _AutoParallelContext is the environment in which operations are executed
@@ -105,11 +118,11 @@ class _AutoParallelContext:
             device_num (int): The device number.
 
         Raises:
-            ValueError: If the device num is not in [1, 4096].
+            ValueError: If the device num is not a positive integer.
         """
         self.check_context_handle()
-        if device_num < 1 or device_num > 4096:
-            raise ValueError("The context configuration parameter 'device_num' must be in [1, 4096], "
+        if device_num < 1:
+            raise ValueError("The context configuration parameter 'device_num' must be a positive integer, "
                              "but got the value of device_num : {}.".format(device_num))
         from mindspore.communication._comm_helper import _HCCL_TEST_AVAILABLE
         self._context_handle.set_hccl_test_avaible(_HCCL_TEST_AVAILABLE)
@@ -229,6 +242,16 @@ class _AutoParallelContext:
         self.check_context_handle()
         return self._context_handle.get_pipeline_stage_split_num()
 
+    def get_pipeline_interleave(self):
+        """Get pipeline interleave flag"""
+        self.check_context_handle()
+        return self._context_handle.get_pipeline_interleave()
+
+    def get_pipeline_scheduler(self):
+        """Get pipeline scheduler"""
+        self.check_context_handle()
+        return self._context_handle.get_pipeline_scheduler()
+
     def set_pipeline_segments(self, segments):
         """Set the segments of the pipeline"""
         if isinstance(segments, bool) or not isinstance(segments, int):
@@ -782,6 +805,57 @@ class _AutoParallelContext:
         self.check_context_handle()
         return self._context_handle.get_enable_fold_pipeline()
 
+    def set_pipeline_config(self, pipeline_config):
+        r"""
+        Set the configuration for pipeline parallelism. The configuration provides more detailed behavior control about
+        parallel training when pipeline parallelism is enabled.
+
+        Args:
+            pipeline_config (dict): The configuration for pipeline parallelism. It supports following keys:
+
+            - pipeline_interleave(bool): Setting true enable interleave scheduler for pipeline parallelism. This
+                                         scheduler requires more memory but less bubble.
+            - pipeline_scheduler(string): There are two choices, "1f1b" and "gpipe". default is "1f1b"
+
+              - 1f1b: It requires less memory and bubble ratio, for it run backward pass when corresponding forward pass
+                      finished.
+              - gpipe: It requires more memory and bubble ratio, for it run backward pass after all forward pass
+                       finished.
+
+        Raises:
+            TypeError: If the type of `pipeline_config` is not `dict`.
+            ValueError: If the key in `pipeline_config` not  in ["pipeline_interleave", "pipeline_scheduler"].
+            ValueError: If pipeline interleave is False, pipeline scheduler is not `1f1b`.
+        """
+        self.check_context_handle()
+
+        if not isinstance(pipeline_config, dict):
+            raise TypeError("For 'set_pipeline_config', the argument 'pipeine_config' "
+                            "must be dict, but got the type : {}.".format(type(pipeline_config)))
+
+        pp_interleave = _PipelineConfig.PIPELINE_INTERLEAVE
+        pp_scheduler = _PipelineConfig.PIPELINE_SCHEDULER
+
+        for config_name in pipeline_config:
+            unknown_config = []
+            if config_name not in [pp_interleave, pp_scheduler]:
+                unknown_config.append(config_name)
+
+            if unknown_config:
+                raise ValueError("Unknown config: {}".format(unknown_config))
+
+        Validator.check_bool(
+            pipeline_config[pp_interleave], pp_interleave, pp_interleave)
+        self._context_handle.set_pipeline_interleave(
+            pipeline_config[pp_interleave])
+
+        Validator.check_string(pipeline_config[pp_scheduler], [_PipelineScheduler.PIPELINE_1F1B,
+                                                               _PipelineScheduler.PIPELINE_GPIPE])
+        if not pipeline_config[pp_interleave] and pipeline_config[pp_scheduler] != _PipelineScheduler.PIPELINE_1F1B:
+            raise ValueError(f"When pipeline_interleave is False, {pp_scheduler} is not supported")
+
+        self._context_handle.set_pipeline_scheduler(pipeline_config[pp_scheduler])
+
     def get_enable_parallel_optimizer(self):
         """Get parallel optimizer flag."""
         self.check_context_handle()
@@ -1068,6 +1142,7 @@ class _AutoParallelContext:
             self.set_enable_all_gather_fusion(openstate)
             self.set_enable_reduce_scatter_fusion(openstate)
 
+
 def _set_ops_strategy_json_config(type="SAVE", path="", mode="all"):
     """
     Set strategy json configuration.
@@ -1091,6 +1166,7 @@ def _set_ops_strategy_json_config(type="SAVE", path="", mode="all"):
     else:
         raise KeyError("Type must be 'SAVE' or 'LOAD' and mode must be 'all' or 'principal'")
 
+
 _AUTO_PARALLEL_CONTEXT = None
 
 
@@ -1126,6 +1202,7 @@ _set_auto_parallel_context_func_map = {
     "dataset_strategy": auto_parallel_context().set_dataset_strategy,
     "enable_parallel_optimizer": auto_parallel_context().set_enable_parallel_optimizer,
     "parallel_optimizer_config": auto_parallel_context().set_parallel_optimizer_config,
+    "pipeline_config": auto_parallel_context().set_pipeline_config,
     "grad_accumulation_step": auto_parallel_context().set_grad_accumulation_step,
     "all_reduce_fusion_config": auto_parallel_context().set_all_reduce_fusion_split_indices,
     "communi_parallel_mode": auto_parallel_context().set_communi_parallel_mode,
@@ -1143,6 +1220,8 @@ _get_auto_parallel_context_func_map = {
     "gradient_fp32_sync": auto_parallel_context().get_gradient_fp32_sync,
     "loss_repeated_mean": auto_parallel_context().get_loss_repeated_mean,
     "pipeline_stages": auto_parallel_context().get_pipeline_stages,
+    "pipeline_interleave": auto_parallel_context().get_pipeline_interleave,
+    "pipeline_scheduler": auto_parallel_context().get_pipeline_scheduler,
     "parallel_mode": auto_parallel_context().get_parallel_mode,
     "search_mode": auto_parallel_context().get_strategy_search_mode,
     "auto_parallel_search_mode": auto_parallel_context().get_auto_parallel_search_mode,

mindspore/parallel/_parallel_serialization.py

@@ -243,6 +243,33 @@ def _extract_pipeline_stage_num(strategy_file):
     return pipeline_stage_num
 
 
+def _extract_src_dst_layout_map_by_src(src_strategy_file=None, dst_strategy_file=None):
+    """Extract strategy list by src strategy"""
+    src_layout_map = _extract_layout_map(src_strategy_file)
+    dst_layout_map = _extract_layout_map(dst_strategy_file)
+    if dst_layout_map is None:
+        return src_layout_map, dst_layout_map
+    for param_name in list(dst_layout_map.keys()):
+        if param_name in src_layout_map.keys():
+            continue
+        dst_layout_map.pop(param_name)
+    stage_id = 0
+    if src_strategy_file[-5:] == ".json":
+        with open(src_strategy_file, 'r') as f:
+            json_content = json.load(f)
+        strategy_items = json_content.get("parallel_strategy_item")
+        if not strategy_items:
+            raise ValueError("The strategy file {} if empty.".format(src_strategy_file))
+        stage_id = strategy_items.get(list(strategy_items.keys())[0]).get('stage')
+    else:
+        src_parallel_strategy_map = _load_protobuf_strategy(src_strategy_file)
+        strategy_items = src_parallel_strategy_map.parallel_strategy_item
+        if not strategy_items:
+            raise ValueError("The strategy file {} if empty.".format(src_strategy_file))
+        stage_id = strategy_items[0].parallel_strategys.stage
+    return src_layout_map, dst_layout_map, stage_id
+
+
 def _extract_src_dst_layout_map(rank_id, src_strategy_file=None, dst_strategy_file=None):
     """Extract strategy list"""
     src_layout_map = _extract_layout_map(src_strategy_file)
@@ -341,6 +368,7 @@ def _transform_parallel_checkpoint(rank_id, param_total_dict, param_attr_dict, s
     Transform model parallel dimension for distributed checkpoint files.
     """
     transform_param_dict = {}
+    device_num = -1
     for param_name, _ in param_total_dict.items():
         tensor_shape = list(param_total_dict[param_name].values())[0].shape
         from_dev_matrix = [1]
@@ -394,14 +422,18 @@ def _transform_parallel_checkpoint(rank_id, param_total_dict, param_attr_dict, s
         to_info_tuple = (to_opt_shard_size, to_dev_matrix_origin, to_tensor_map_origin, origin_tensor_shape)
         _insert_opt_shard_reshape(param_rank_map, from_info_tuple, to_info_tuple)
         transform_operator_stack = _generate_transform_operator_stack(param_rank_map, rank_id)
-        _apply_tensor_transform_operators(transform_operator_stack, param_total_dict[param_name], device_num)
-        transform_tensor = ms.Tensor(param_total_dict[param_name][rank_id % device_num])
+        param_total_dict_copy = param_total_dict[param_name].copy()
+        _apply_tensor_transform_operators(transform_operator_stack, param_total_dict_copy, device_num)
+        transform_tensor = ms.Tensor(param_total_dict_copy[rank_id % device_num])
         requires_grad = param_attr_dict[param_name][rank_id % device_num][0]
         layerwise_parallel = param_attr_dict[param_name][rank_id % device_num][1]
         transform_para = ms.Parameter(transform_tensor, param_name, requires_grad, layerwise_parallel)
         if param_type_dict[param_name][rank_id % device_num] == "BFloat16":
             transform_para.set_dtype(ms.bfloat16)
         transform_param_dict[param_name] = transform_para
+    if device_num < 0:
+        raise ValueError("None of the parameters in checkpoint file are in either src strategy or "
+                         "dst strategy. Please check correctness of strategy files.")
 
     # Handle those parameter like learning_rate, global_step which not in strategy_file.
     for param_name, _ in param_total_dict.items():

mindspore/parallel/_tensor.py

@@ -223,7 +223,9 @@ def _load_tensor(tensor, dev_mat, tensor_map, rank_id=-1):
     tensor_strategy = _get_tensor_strategy(dev_mat, tensor_map)
     tensor_slice_index = _get_tensor_slice_index(dev_mat, tensor_strategy, tensor_map, rank)
     if tensor.dtype == mstype.bfloat16:
-        tensor = tensor.float()
+        from mindspore.ops.operations import Cast
+        cpu_cast = Cast().set_device("CPU")
+        tensor = cpu_cast(tensor, mstype.float32)
     np_tensor = tensor.asnumpy()
     np_tensor_list = _chunk_tensor_by_strategy(np_tensor, tensor_strategy)
     np_tensor_slice = np_tensor_list[int(tensor_slice_index)]

mindspore/parallel/_transformer/transformer.py

@@ -805,14 +805,14 @@ class MultiHeadAttention(Cell):
             - **attention_mask** (Tensor) - If the use_past is False or is_first_iteration=True, the attention mask
               matrix should ba (batch_size, src_seq_length, tgt_seq_length), or None. None means there will be no mask
               in softmax computation. Otherwise, the mask must be (batch_size, 1, tgt_seq_length)
-            - **key_past** (Tensor) - Float16 tensor with shape (batch_size, num_heads, size_per_head, tgt_seq_length).
+            - **key_past** (Tensor) - float16 tensor with shape (batch_size, num_heads, size_per_head, tgt_seq_length).
               The past calculated key vector. Used for incremental prediction when the use_past is True.
               Default None.
-            - **value_past** (Tensor) - Float16 tensor with shape
+            - **value_past** (Tensor) - float16 tensor with shape
               (batch_size, num_heads, tgt_seq_length, size_per_head).
               The past calculated value vector. Used for incremental prediction when the use_past is True.
               Default None.
-            - **batch_valid_length** (Tensor) - Int32 tensor with shape (batch_size,) the past calculated the index.
+            - **batch_valid_length** (Tensor) - int32 tensor with shape (batch_size,) the past calculated the index.
               Used for incremental prediction when the use_past is True. Default None.
 
         Outputs:
@@ -1412,7 +1412,7 @@ class TransformerEncoderLayer(Cell):
               be no mask in softmax computation. Otherwise, should be [batch_size, 1, hidden_size]
             - **init_reset** (Tensor) - A bool tensor with shape [1], used to clear the past key parameter and
               past value parameter used in the incremental prediction. Only valid when use_past is True. Default True.
-            - **batch_valid_length** (Tensor) - Int32 tensor with shape [batch_size] the past calculated the index.
+            - **batch_valid_length** (Tensor) - int32 tensor with shape [batch_size] the past calculated the index.
               Used for incremental prediction when the use_past is True. Default None.
 
         Outputs:
@@ -1824,7 +1824,7 @@ class TransformerDecoderLayer(Cell):
               means there will be no mask in softmax computation in cross attention. Default None.
             - **init_reset** (Tensor) - A bool tensor with shape [1], used to clear the past key parameter and
               past value parameter used in the incremental prediction. Only valid when use_past is True. Default True.
-            - **batch_valid_length** (Tensor) - Int32 tensor with shape [batch_size] the past calculated the index.
+            - **batch_valid_length** (Tensor) - int32 tensor with shape [batch_size] the past calculated the index.
               Used for incremental prediction when the use_past is True. Default None.
 
         Outputs:
@@ -2333,7 +2333,7 @@ class TransformerEncoder(Cell):
               be no mask in softmax computation. Otherwise, should be [batch_size, 1, hidden_size]
             - **init_reset** (Tensor) - A bool tensor with shape [1], used to clear the past key parameter and
               past value parameter used in the incremental prediction. Only valid when use_past is True. Default True.
-            - **batch_valid_length** (Tensor) - Int32 tensor with shape [batch_size] the past calculated the index.
+            - **batch_valid_length** (Tensor) - int32 tensor with shape [batch_size] the past calculated the index.
               Used for incremental prediction when the use_past is True. Default None.
 
         Outputs:
@@ -2589,7 +2589,7 @@ class TransformerDecoder(Cell):
               means there will be no mask in softmax computation in cross attention. Default None.
             - **init_reset** (Tensor) - A bool tensor with shape [1], used to clear the past key parameter and
               past value parameter used in the incremental prediction. Only valid when use_past is True. Default True.
-            - **batch_valid_length** (Tensor) - Int32 tensor with shape [batch_size] the past calculated the index.
+            - **batch_valid_length** (Tensor) - int32 tensor with shape [batch_size] the past calculated the index.
               Used for incremental prediction when the use_past is True. Default None.
 
         Outputs:
@@ -2842,7 +2842,7 @@ class Transformer(Cell):
               seq_length, hidden_size], this should be none if the decoder layer is 0 or the user wants no mask.
             - **init_reset** (Tensor) - A bool tensor with shape [1], used to clear the past key parameter and
               past value parameter used in the incremental prediction. Only valid when use_past is True. Default True.
-            - **batch_valid_length** (Tensor) - Int32 tensor with shape [batch_size] the past calculated the index.
+            - **batch_valid_length** (Tensor) - int32 tensor with shape [batch_size] the past calculated the index.
               Used for incremental prediction when the use_past is True. Default None.
 
         Outputs: