mindspore 2.7.0__cp311-cp311-win_amd64.whl → 2.7.1__cp311-cp311-win_amd64.whl

mindspore/runtime/memory.py

@@ -14,9 +14,15 @@
 # ============================================================================
 
 """Memory interfaces."""
+import contextlib
+import ctypes
 import os
 from mindspore._c_expression import RuntimeConf, DeviceManagerConf, _memory_stats, \
     _reset_max_mem_reserved, _reset_max_mem_allocated, DeviceContextManager, _empty_cache, _memory_replay
+try:
+    from mindspore._c_expression import _enable_pluggable_allocator, _disable_pluggable_allocator
+except ImportError:
+    pass
 from mindspore import _checkparam as Validator
 from mindspore._checkparam import args_type_check
 from mindspore import log as logger
@@ -406,3 +412,109 @@ def memory_replay(file_path):
         >>> ms.runtime.memory_replay("/data/memory_block.csv")
     """
     _memory_replay(os.path.realpath(file_path))
+
+
+class PluggableAllocator():
+    r"""
+    Receive a .so file via ctypes, and dynamically load the alloc and free functions within it.
+    It needs to be used in conjunction with :class:`mindspore.runtime.MemPool` and
+    :func:`mindspore.runtime.use_mem_pool` to take over the memory allocation and free
+    in the MindSpore memory pool.
+
+    .. warning::
+        This is currently supported only in unix OSs.
+
+    Args:
+        path_to_so_file(str): Path in the file system to the `.so` file containing
+            the allocator functions.
+        alloc_fn_name(str): Name of the function to perform the memory allocation
+            in the so file. The signature must be:
+            `void* alloc_fn(size_t size, int device, aclrtStream stream);` .
+        free_fn_name(str): Name of the function to perform the memory release
+            in the so file. The signature must be:
+            `void free_fn(void* ptr, size_t size, aclrtStream stream);` .
+
+    Supported Platforms:
+        ``Ascend``
+    """
+
+    def __init__(self, path_to_so_file: str, alloc_fn_name: str, free_fn_name: str):
+        allocator = ctypes.CDLL(path_to_so_file)
+        alloc_fn = ctypes.cast(getattr(allocator, alloc_fn_name), ctypes.c_void_p).value
+        free_fn = ctypes.cast(getattr(allocator, free_fn_name), ctypes.c_void_p).value
+        if alloc_fn is None:
+            raise ValueError(f"Cannot find allocator function {alloc_fn_name} in {path_to_so_file}")
+        if free_fn is None:
+            raise ValueError(f"Cannot find free function {free_fn_name} in {path_to_so_file}")
+        self._alloc_fn = alloc_fn
+        self._free_fn = free_fn
+
+    @property
+    def alloc_fn_ptr(self) -> int:
+        """Function pointer of the allocator function."""
+        return self._alloc_fn
+
+    @property
+    def free_fn_ptr(self) -> int:
+        """Function pointer of the free function."""
+        return self._free_fn
+
+
+class MemPool():
+    r"""
+    A MemPool warp a :class:`mindspore.runtime.PluggableAllocator`,
+    and pass it to :func:`mindspore.runtime.use_mem_pool`.
+
+    Args:
+        allocator(mindspore.runtime.PluggableAllocator): a mindspore.runtime.PluggableAllocator
+            that can be used to define how memory gets allocated and freed in the pool.
+
+    Supported Platforms:
+        ``Ascend``
+    """
+
+    def __init__(self, allocator: PluggableAllocator):
+        self._allocator = allocator
+
+    @property
+    def allocator(self) -> PluggableAllocator:
+        """The allocator used by the pool."""
+        return self._allocator
+
+
+@contextlib.contextmanager
+def use_mem_pool(pool: MemPool):
+    r"""
+    A context manager that routes allocations and deallocations to a given pool.
+
+    Note:
+        - This context manager makes only current thread's allocations route to the given pool.
+        - If a new thread is spawned inside the context manager the allocations in that thread
+          will not route to the given pool.
+        - Only by allocating Device memory inside the context manager, the allocation operation
+          can be routed to the given pool.
+        - Only Atlas A2 training series products support this interface.
+
+    Args:
+        pool(mindspore.runtime.MemPool): a MemPool object that warp a PluggableAllocator.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> import mindspore as ms
+        >>> path = "/path/to/allocator.so"
+        >>> allocator = ms.runtime.PluggableAllocator(path, "Alloc", "Free")
+        >>> mem_pool = ms.runtime.MemPool(allocator)
+        >>> shape = (1024, 1024)
+        >>> x = ms.ops.Ones()(shape, ms.float32)
+        >>> with ms.runtime.use_mem_pool(mem_pool):
+        >>>     y = ms.ops.Ones()(shape, ms.float32)
+        >>> output = x + y
+    """
+    allocator = pool.allocator
+    _enable_pluggable_allocator(allocator.alloc_fn_ptr, allocator.free_fn_ptr)
+    try:
+        yield
+    finally:
+        _disable_pluggable_allocator()

mindspore/{experimental/llm_boost → tools}/__init__.py

@@ -12,11 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ============================================================================
-"""LlmBoost Register"""
+"""Tools module."""
 from __future__ import absolute_import
 
-from mindspore.experimental.llm_boost.atb import LlamaBoost, QwenBoost
-from mindspore.experimental.llm_boost.ascend_native import *
-from mindspore.experimental.llm_boost.register import LlmBoostRegister
+__all__ = ["stress_detect", "sdc_detect_start", "sdc_detect_stop", "get_sdc_detect_result", "set_dump"]
 
-__all__ = ["LlmBoostRegister"]
+from .stress_detect import stress_detect
+from .sdc_detect import sdc_detect_start, sdc_detect_stop, get_sdc_detect_result
+from .data_dump import set_dump

mindspore/tools/data_dump.py

@@ -0,0 +1,130 @@
+# Copyright 2021-2025 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Controlling dump behavior."""
+from __future__ import absolute_import
+from warnings import warn
+
+import mindspore.context as context
+from mindspore._c_expression import security
+
+
+def set_dump(target, enabled=True):
+    """
+    Enable or disable dump for the `target` and its contents.
+
+    `target` should be an instance of :class:`mindspore.nn.Cell` or :class:`mindspore.ops.Primitive` .
+    Please note that this API takes effect only when the Dump function is enabled, and the `dump_mode`
+    field in the Dump configuration file is set to `"2"` with the `ms_backend` compilation backend
+    (please refer to the backend parameter in
+    `jit <https://www.mindspore.cn/docs/en/master/api_python/mindspore/mindspore.jit.html>`_).
+    See the `dump document <https://www.mindspore.cn/tutorials/en/master/debug/dump.html>`_ for details.
+    By default, instances of :class:`mindspore.nn.Cell` and :class:`mindspore.ops.Primitive` do not enable
+    the Dump data feature.
+
+    Note:
+        1. This API is only available for JIT compilation, requires 'Ascend' as the device_target and
+           `ms_backend` as the compilation backend (please refer to the backend parameter in
+           `jit <https://www.mindspore.cn/docs/en/master/api_python/mindspore/mindspore.jit.html>`_),
+           and does not support fused operators.
+        2. This API only supports being called before training starts.
+           If you call this API during training, it may not be effective.
+        3. After using `set_dump(Cell, True)` , operators in forward and backward
+           computation  (computation generated by the grad operations) of the
+           cell will be dumped.
+        4. For :class:`mindspore.nn.SoftmaxCrossEntropyWithLogits` layer, the forward
+           computation and backward computation use the same set of
+           operators. So you can only see dump data from backward computation.
+           Please note that :class:`mindspore.nn.SoftmaxCrossEntropyWithLogits` layer will also use
+           the above operators internally when initialized with `sparse=True` and
+           `reduction="mean"` .
+
+    Args:
+        target (Union[Cell, Primitive]): The Cell instance or Primitive instance
+            to which the dump flag is set.
+        enabled (bool, optional): ``True`` indicates that the dump is enabled, and ``False`` indicates that
+            the dump is disabled.
+            Default: ``True`` .
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        .. note::
+            Please set environment variable `MINDSPORE_DUMP_CONFIG` to the dump config file and set `dump_mode` field
+            in dump config file to 2 before running this example.
+            See `dump document <https://www.mindspore.cn/tutorials/en/master/debug/dump.html>`_ for details.
+
+        >>> import numpy as np
+        >>> import mindspore as ms
+        >>> import mindspore.nn as nn
+        >>> from mindspore import Tensor, jit
+        >>> from mindspore.tools import set_dump
+        >>>
+        >>> ms.set_device(device_target="Ascend")
+        >>>
+        >>> class MyNet(nn.Cell):
+        ...     def __init__(self):
+        ...         super().__init__()
+        ...         self.conv1 = nn.Conv2d(5, 6, 5, pad_mode='valid')
+        ...         self.relu1 = nn.ReLU()
+        ...
+        ...     @jit
+        ...     def construct(self, x):
+        ...         x = self.conv1(x)
+        ...         x = self.relu1(x)
+        ...         return x
+        >>>
+        >>> if __name__ == "__main__":
+        ...     net = MyNet()
+        ...     set_dump(net.conv1)
+        ...     input_tensor = Tensor(np.ones([1, 5, 10, 10], dtype=np.float32))
+        ...     output = net(input_tensor)
+    """
+    if security.enable_security():
+        raise ValueError('The set_dump API is not supported, please recompile '
+                         'source without "-s on".')
+
+    import mindspore.nn as nn  # avoid circular import
+    from mindspore.ops import Primitive
+    if not isinstance(target, nn.Cell) and not isinstance(target, Primitive):
+        raise ValueError(f"The \"target\" parameter must be an instance of "
+                         f"Cell or Primitive, "
+                         f"but got an instance of {type(target)}.")
+
+    if not isinstance(enabled, bool):
+        raise ValueError("The \"enabled\" parameter must be bool.")
+
+    # Checking for device target and mode.
+    current_target = context.get_context("device_target")
+    if current_target != "Ascend":
+        # We will not return here in case user changed device_target later.
+        warn("Current device_target is {}, which is not supported by set_dump. "
+             "Only Ascend device target is supported currently. "
+             "If you have Ascend device, consider set device_target to Ascend "
+             "before calling set_dump.".format(current_target))
+
+    # The actual set dump logic.
+    if isinstance(target, nn.Cell):
+        target.add_flags(dump=enabled)
+        for cell in target.cells():
+            set_dump(cell, enabled)
+
+        primitives = getattr(target, "_primitives", {})
+        for value in primitives.values():
+            if value and "dump" in value.attrs:
+                set_dump(value, enabled)
+
+    if isinstance(target, Primitive):
+        target.add_prim_attr("dump", "true" if enabled else "false")

mindspore/tools/sdc_detect.py

@@ -0,0 +1,91 @@
+# Copyright 2025 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""SDC detect."""
+from mindspore import _c_expression
+
+
+def sdc_detect_start():
+    """
+    Start silent data corruption detection. It will check the inputs and outputs of MatMul operations during the
+    forward and backward computations on the current device, which may increase execution time. The overhead of the
+    check time decreases as the matrix shapes increase. Starting sdc detection results in approximately 100%
+    performance degradation for a single 4096-sized MatMul computation, and approximately 90% degradation on the
+    Llama2-7B model (model parallel is 4, pipeline parallel is 2, and using qkv concatenation and ffn concatenation in
+    decoder layers).
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> from mindspore.tools import sdc_detect_start
+        >>> sdc_detect_start()
+    """
+    _c_expression.sdc_detect_start()
+
+
+def sdc_detect_stop():
+    """
+    Stop silent data corruption detection.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> from mindspore.tools import sdc_detect_stop
+        >>> sdc_detect_stop()
+    """
+    _c_expression.sdc_detect_stop()
+
+
+def get_sdc_detect_result():
+    """
+    Get the result of silent data corruption detection.
+
+    Returns:
+        bool, indicating whether silent data corruption has occurred after detection start.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> from mindspore.tools import get_sdc_detect_result
+        >>> result = get_sdc_detect_result()
+        >>> print(result)
+        False
+    """
+    return _c_expression.get_sdc_detect_result()
+
+
+class _SdcDetector:
+    """
+    Manager of feature value sampling for SDC detect
+    """
+    def __init__(self):
+        self.param_count = -1
+
+    def need_sample(self):
+        """"If need to sample feature value."""
+        if not _c_expression.is_silent_detect_enable():
+            return False
+        grad_sample_interval = _c_expression.get_silent_detect_config('grad_sample_interval')
+        self.param_count = (self.param_count + 1) % grad_sample_interval
+        return self.param_count == 0
+
+    @staticmethod
+    def get_dump_name(param_name):
+        """Get dump file name with sdc prefix."""
+        return _c_expression.get_silent_detect_feature_name(param_name)
+
+_sdc_detector = _SdcDetector()

mindspore/tools/stress_detect.py

@@ -0,0 +1,63 @@
+# Copyright 2025 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Stress detect."""
+from mindspore import _c_expression
+from mindspore import log as logger
+from mindspore.communication import init, create_group, get_rank
+from mindspore.communication import get_local_rank_size
+
+
+def stress_detect(detect_type="aic"):
+    """
+    Used to detect whether there are faults in hardware accuracy or communication between links.
+    The common usage scenario is to initiate a new thread or call this interface through a Callback function
+    at each step or when saving checkpoints, to check whether hardware malfunctions could affect accuracy.
+
+    Args:
+        detect_type (str, optional): The type of stress test to perform. There are two options available: ``'aic'`` and
+            ``'hccs'``, which perform AiCore and HCCS link stress tests on the device, respectively. Default: "aic".
+
+    Returns:
+        int, the return value represents the error type. 0 indicates normal. 1 indicates failure to start some or
+        all test cases. 2 indicates a hardware failure, and it is recommended to replace the device.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> from mindspore.tools import stress_detect
+        >>> ret = stress_detect()
+        >>> print(ret)
+        0
+    """
+    if detect_type not in ["aic", "hccs"]:
+        logger.error(f"For stress detect, detection type must be 'aic' or 'hccs'."
+                     f"But got {detect_type}. Exiting stress detect.")
+        return 1
+
+    if detect_type == "aic":
+        return _c_expression.stress_detect("aic")
+
+    init()
+    local_ranks = []
+    local_rank_size = get_local_rank_size()
+    node_num = get_rank() // local_rank_size
+    for i in range(local_rank_size):
+        local_ranks.append(local_rank_size * node_num + i)
+    if get_rank() in local_ranks:
+        group = f"new_group_{node_num}"
+        create_group(group, local_ranks)
+
+    return _c_expression.stress_detect(group)

mindspore/train/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Huawei Technologies Co., Ltd
+# Copyright 2025 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -25,8 +25,8 @@ from mindspore.train import amp
 from mindspore.train.amp import build_train_network
 from mindspore.train.loss_scale_manager import LossScaleManager, FixedLossScaleManager, DynamicLossScaleManager
 from mindspore.train.serialization import save_checkpoint, load_checkpoint, load_param_into_net, export, \
-    load, parse_print, async_ckpt_thread_status, convert_model, export_split_mindir, \
-    load_checkpoint_async, check_checkpoint, get_ckpt_path_with_strategy, ckpt_to_safetensors, safetensors_to_ckpt, \
+    load, async_ckpt_thread_status, export_split_mindir, \
+    load_checkpoint_async, get_ckpt_path_with_strategy, ckpt_to_safetensors, safetensors_to_ckpt, \
     build_searched_strategy, merge_sliced_parameter, load_distributed_checkpoint, restore_group_info_list
 from mindspore.train.callback import Callback, LossMonitor, TimeMonitor, ModelCheckpoint, SummaryCollector, \
     CheckpointConfig, RunContext, LearningRateScheduler, SummaryLandscape, FlopsUtilizationCollector, \
@@ -37,9 +37,9 @@ from mindspore.train.metrics import *
 from mindspore.train.data_sink import data_sink
 
 __all__ = ["Model", "DatasetHelper", "connect_network_with_dataset", "build_train_network", "LossScaleManager",
-           "FixedLossScaleManager", "DynamicLossScaleManager", "save_checkpoint", "load_checkpoint", "check_checkpoint",
-           "load_param_into_net", "export", "load", "export_split_mindir", "parse_print", "async_ckpt_thread_status",
-           "convert_model", "data_sink", "load_checkpoint_async", "get_ckpt_path_with_strategy", "ckpt_to_safetensors",
+           "FixedLossScaleManager", "DynamicLossScaleManager", "save_checkpoint", "load_checkpoint",
+           "load_param_into_net", "export", "load", "export_split_mindir", "async_ckpt_thread_status",
+           "data_sink", "load_checkpoint_async", "get_ckpt_path_with_strategy", "ckpt_to_safetensors",
            "safetensors_to_ckpt", "build_searched_strategy", "merge_sliced_parameter", "load_distributed_checkpoint",
            "restore_group_info_list"]
 __all__.extend(callback.__all__)

mindspore/train/_utils.py

@@ -344,15 +344,7 @@ def _get_layout_opt_shard(layout_obj, param_redundancy_dict):
     """Layout ckpt append opt shard."""
     for key, value in layout_obj.items():
         if value[5]:
-            world_groups = ("hccl_world_group", "nccl_world_group", "mccl_world_group")
-            if value[5] in world_groups:
-                opt_para_num = get_group_size()
-            elif "-" in value[5]:
-                opt_para_str = value[5].split("-")[0]
-                opt_para_num = int(opt_para_str)
-            else:
-                raise ValueError(f"For get_parameter_redundancy, the format of the parallel communication domain for "
-                                 f"the optimizer is incorrect.")
+            opt_para_num = get_group_size(value[5])
             param_redundancy_ranks = param_redundancy_dict.get(key)
             res = []
             for param_ranks in param_redundancy_ranks:
@@ -582,17 +574,12 @@ def _progress_bar(iterable, total=None):
         print_progress_bar(i)
 
 
-def _load_and_transform(path, name_map, load_func, transform_func=None):
+def _load_and_transform(path, name_map, load_func):
     """use load_func to load and use transform_func to convert"""
-    if load_func is not None:
-        param_dict = load_func(path)
-    else:
-        param_dict = path
+    param_dict = load_func(path)
     transform_dict = {}
+
     for k, v in param_dict.items():
         new_name = name_map.get(k, k) if name_map is not None else k
-        if transform_func is not None:
-            transform_dict[new_name] = transform_func(v, new_name)
-        else:
-            transform_dict[new_name] = v
+        transform_dict[new_name] = v
     return transform_dict

mindspore/train/amp.py

@@ -818,8 +818,10 @@ def get_white_list():
          <class 'mindspore.ops.operations.nn_ops.Conv2DTranspose'>,
          <class 'mindspore.ops.operations.nn_ops.Conv3DTranspose'>,
          <class 'mindspore.ops.operations.nn_ops.Conv2DBackpropInput'>,
-         <class 'mindspore.ops.operations.math_ops.MatMul'>, <class 'mindspore.ops.operations.math_ops.BatchMatMul'>,
-         <class 'mindspore.ops.operations.nn_ops.PReLU'>, <class 'mindspore.ops.operations.nn_ops.ReLU'>,
+         <class 'mindspore.ops.auto_generate.gen_ops_prim.MatMul'>,
+         <class 'mindspore.ops.auto_generate.gen_ops_prim.BatchMatMul'>,
+         <class 'mindspore.ops.auto_generate.gen_ops_prim.PReLU'>,
+         <class 'mindspore.ops.auto_generate.gen_ops_prim.ReLU'>,
          <class 'mindspore.ops.operations.math_ops.Ger'>]
     """
     white_list = AMP_WHITE_LIST.copy()
@@ -871,8 +873,8 @@ def custom_mixed_precision(network, *, white_list=None, black_list=None, dtype=m
             white list is not used.
         black_list (list[Cell], optional): Black list of custom mixed precision. Defaults: ``None`` , means
             black list is not used.
-        dtype (Type): The type used in lower precision calculations, can be ``mstype.float16`` or ``mstype.bfloat16`` ,
-            default: ``mstype.float16`` .
+        dtype (Type, optional): The type used in lower precision calculations, can be ``mstype.float16`` or
+            ``mstype.bfloat16`` , default: ``mstype.float16`` .
 
     Returns:
         network (Cell), A network supporting mixed precision.

mindspore/train/callback/_checkpoint.py

@@ -27,7 +27,6 @@ from mindspore.train._utils import _make_directory
 from mindspore.train.serialization import save_checkpoint, _save_graph, _wait_async_process_save_ckpt, \
     _wait_async_thread_save_ckpt, _check_async_save
 from mindspore.parallel._cell_wrapper import destroy_allgather_cell
-from mindspore.parallel._recovery_context import _set_recovery_context, _get_recovery_context
 from mindspore.communication.management import get_rank, get_group_size
 from mindspore.train._utils import get_parameter_redundancy, remove_param_redundancy, _get_pp_size_from_redundancy_map
 from mindspore.train.callback._callback import Callback
@@ -509,9 +508,6 @@ class ModelCheckpoint(Callback):
         if callable(prefix):
             self._prefix_func = prefix
 
-        if context.get_context("device_target") == "GPU" and _get_recovery_context("enable_recovery"):
-            _set_recovery_context(ckpt_path=self._directory)
-
         if config is None:
             self._config = CheckpointConfig()
         else:
@@ -577,11 +573,6 @@ class ModelCheckpoint(Callback):
             self._directory = self._directory_func(cb_params)
             _make_directory(self._directory)
         collect_host_info("Callback", "ModelCheckpoint", "step_end", start_time=get_clock_syscnt(), level=1)
-        # In disaster recovery scenario, the training process may be rolled back to the last step where
-        # the ckpt was successfully saved, so the _last_triggered_step should be updated.
-        if _get_recovery_context("enable_recovery") and cb_params.last_save_ckpt_step is not None:
-            self._last_triggered_step = cb_params.last_save_ckpt_step
-            cb_params.last_save_ckpt_step = None
 
         # save graph (only once)
         if not self._graph_saved: