mindspore 2.5.0__cp311-cp311-win_amd64.whl → 2.6.0__cp311-cp311-win_amd64.whl

mindspore/train/summary/_summary_adapter.py

@@ -25,7 +25,7 @@ from PIL import Image
 
 from mindspore import log as logger
 from mindspore import context
-from mindspore.communication.management import get_rank
+from mindspore.communication.management import get_rank, get_group_size
 from mindspore.communication.management import GlobalComm
 
 from mindspore import _checkparam as Validator
@@ -60,7 +60,7 @@ def get_event_file_name(prefix, suffix, time_second):
     file_name = ""
     hostname = platform.node()
 
-    device_num = context.get_auto_parallel_context('device_num')
+    device_num = get_group_size()
     device_id = context.get_context('device_id')
     if device_num > 1 or GlobalComm.WORLD_COMM_GROUP == 'nccl_world_group':
         # Notice:

mindspore/train/summary/summary_record.py

@@ -26,7 +26,7 @@ from mindspore import log as logger
 from mindspore.nn import Cell
 from mindspore import context
 from mindspore._c_expression import security
-from mindspore._c_expression import Tensor as Tensor_
+from mindspore._c_expression import TensorPy as Tensor_
 from mindspore.common import dtype as mstype
 from mindspore.common.tensor import Tensor
 from mindspore import _checkparam as Validator

mindspore/utils/__init__.py

@@ -15,9 +15,10 @@
 """Utils module."""
 from __future__ import absolute_import
 from mindspore._c_expression import stress_detect, _reuse_data_ptr
-from .utils import ExitByRequest
+from .utils import ExitByRequest, _tft_handler
 from .runtime_execution_order_check import runtime_execution_order_check
 from . import dryrun
 
 # Symbols from utils module.
-__all__ = ["stress_detect", "ExitByRequest", "runtime_execution_order_check", "dryrun", "_reuse_data_ptr"]
+__all__ = ["stress_detect", "ExitByRequest", "runtime_execution_order_check", "dryrun", "_reuse_data_ptr",
+           "_tft_handler"]

mindspore/utils/dryrun.py

@@ -15,7 +15,6 @@
 """dryrun."""
 import traceback
 import os
-from mindspore._c_expression import Tensor as Tensor_
 from mindspore.common import Tensor
 from mindspore import log as logger
 from mindspore.common._stub_tensor import StubTensor
@@ -52,7 +51,7 @@ def no_inject_traceback_for_print(self):
         return "Unknown Tensor type!"
     if self.has_init:
         self.init_data()
-    return str(Tensor_.asnumpy(self))
+    return str(Tensor.asnumpy(self))
 
 
 def set_simulation():
@@ -85,6 +84,9 @@ def set_simulation():
     StubTensor.flush_from_cache = obj.inject(StubTensor.flush_from_cache)
     Tensor.__str__ = no_inject_traceback_for_print
     StubTensor.__str__ = no_inject_traceback_for_print
+    Tensor.tolist = obj.inject(Tensor.tolist)
+    Tensor.__int__ = obj.inject(Tensor.__int__)
+    Tensor.__float__ = obj.inject(Tensor.__float__)
 
 
 def mock(mock_val, *args):

mindspore/utils/hooks.py

@@ -0,0 +1,81 @@
+# Copyright 2025 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""hooks"""
+from collections import OrderedDict
+import weakref
+from typing import Any, Tuple
+
+
+class _RemovableHandle:
+    r"""
+    A handle which provides the capability to remove a hook.
+
+    Args:
+        hooks_dict (dict): A dictionary of hooks, indexed by hook `id`.
+
+    Keyword Args:
+        extra_dict (Union[dict, list[dict]], optional): An additional dictionary or list of
+            dictionaries whose keys will be deleted when the same keys are
+            removed from `hooks_dict`. Default ``None``.
+    """
+
+    id: int
+    next_id: int = 0
+
+    def __init__(self, hooks_dict: Any, *, extra_dict: Any = None) -> None:
+        self.hooks_dict_ref = weakref.ref(hooks_dict)
+        self.id = _RemovableHandle.next_id
+        _RemovableHandle.next_id += 1
+
+        self.extra_dict_ref: Tuple = ()
+        if isinstance(extra_dict, dict):
+            self.extra_dict_ref = (weakref.ref(extra_dict),)
+        elif isinstance(extra_dict, list):
+            self.extra_dict_ref = tuple(weakref.ref(d) for d in extra_dict)
+
+    def remove(self) -> None:
+        hooks_dict = self.hooks_dict_ref()
+        if hooks_dict is not None and self.id in hooks_dict:
+            del hooks_dict[self.id]
+
+        for ref in self.extra_dict_ref:
+            extra_dict = ref()
+            if extra_dict is not None and self.id in extra_dict:
+                del extra_dict[self.id]
+
+    def __getstate__(self):
+        if self.extra_dict_ref is None:
+            return (self.hooks_dict_ref(), self.id)
+        return (self.hooks_dict_ref(), self.id, tuple(ref() for ref in self.extra_dict_ref))
+
+    def __setstate__(self, state) -> None:
+        if state[0] is None:
+            # create a dead reference
+            self.hooks_dict_ref = weakref.ref(OrderedDict())
+        else:
+            self.hooks_dict_ref = weakref.ref(state[0])
+        self.id = state[1]
+        _RemovableHandle.next_id = max(_RemovableHandle.next_id, self.id + 1)
+
+        if len(state) < 3 or state[2] is None:
+            self.extra_dict_ref = ()
+        else:
+            self.extra_dict_ref = tuple(weakref.ref(d) for d in state[2])
+
+    def __enter__(self) -> "_RemovableHandle":
+        return self
+
+    def __exit__(self, type: Any, value: Any, tb: Any) -> None:
+        self.remove()

mindspore/utils/runtime_execution_order_check.py

@@ -25,6 +25,8 @@ import mindspore.log as logger
 
 # Set Recursion Depth Limit
 sys.setrecursionlimit(10000)
+# support hccl group 150000 card
+csv.field_size_limit(1024 * 1024)
 
 
 class ExecuteOrder:

mindspore/utils/utils.py

@@ -15,13 +15,17 @@
 """mindspore utils."""
 from __future__ import absolute_import
 
+import os
+from mindspore import log as logger
+from mindspore import context
+from mindspore import _checkparam as Validator
 from mindspore.common import dtype as mstype
 from mindspore.common.tensor import Tensor
 from mindspore.ops import functional as F
 from mindspore.ops import operations as P
-from mindspore.context import ParallelMode
-from mindspore.parallel._utils import _get_parallel_mode
+from mindspore.parallel._recovery_context import _set_recovery_context
 from mindspore.common.api import jit_class
+from mindspore._c_expression import _tft_start_record_threads, _tft_finish_record_threads
 
 
 @jit_class
@@ -37,9 +41,9 @@ class ExitByRequest:
         self.equal = P.Equal()
         self.assign = P.Assign()
         self.reduce_all = P.ReduceAll(keep_dims=False)
-        self.is_distributed = _get_parallel_mode() != ParallelMode.STAND_ALONE
+        self.group_size = get_group_size()
+        self.is_distributed = self.group_size > 1
         if self.is_distributed:
-            self.group_size = get_group_size()
             self.base = Tensor([self.group_size], dtype=mstype.int32)
         self.base1 = Tensor([1], mstype.int32)
         self.true = Tensor(True, mstype.bool_)
@@ -58,3 +62,133 @@ class ExitByRequest:
             reduce_all = self.reduce_all(equal)
             grad = F.depend(grad, self.assign(exit_value, reduce_all))
         return grad
+
+
+class TftHandle:
+    """TftHandle class"""
+
+    def __init__(self):
+        super(TftHandle, self).__init__()
+        self._controller_ip = None
+        self._controller_rank_id = None
+        self._controller_port = None
+        self.tft = None
+        self.enable_mindx = False
+
+    def get_tft(self):
+        """return tft handle"""
+        return self.tft
+
+    def unregister_tft(self):
+        """unregister tft"""
+        cur_rank = int(os.getenv("MS_NODE_ID"))  # from msrun
+        if cur_rank == self._controller_rank_id and not self.enable_mindx:
+            self.tft.tft_destroy_controller()
+        self.tft.tft_destroy_processor()
+
+    def _mindx_stub(self):
+        """stub func for mindx"""
+        from mindio_ttp.controller_ttp import (tft_register_mindx_callback,
+                                               tft_notify_controller_stop_train,
+                                               tft_notify_controller_on_global_rank,
+                                               tft_notify_controller_change_strategy)
+
+        def report_fault_ranks_func(error_rank_dict):
+            tft_notify_controller_stop_train(error_rank_dict)
+            return 0
+
+        def report_stop_complete_func(code, msg, error_rank_dict):
+            tft_notify_controller_on_global_rank(error_rank_dict)
+            return 0
+
+        def report_strategies_func(error_rank_dict, strategy_list):
+            tft_notify_controller_change_strategy(strategy_list[-1])
+            return 0
+
+        def report_result(code, msg, error_rank_dict, curr_strategy):
+            if code != 0:
+                tft_notify_controller_change_strategy('dump')
+            return 0
+
+        logger.warning('Stub for mindx.')
+        tft_register_mindx_callback('report_fault_ranks', report_fault_ranks_func)
+        tft_register_mindx_callback('report_stop_complete', report_stop_complete_func)
+        tft_register_mindx_callback('report_strategies', report_strategies_func)
+        tft_register_mindx_callback('report_result', report_result)
+        logger.warning('Stub register mindx func success.')
+
+    def init(self, config=None, **kwargs):
+        """
+        TFT handle init fun. Mainly used to initialize the mindio component.
+
+        Args:
+            config (Dict): configs or the training model running. If ARF is not used, this parameter can be None.
+            **kwargs: Reserved parameters.
+        """
+        tft_env = os.getenv("MS_ENABLE_TFT", "")
+        if ("TTP:1" not in tft_env) and ("UCE:1" not in tft_env) and ("ARF:1" not in tft_env):
+            raise ValueError("MindIO TFT regitster need custom switch on[MS_ENABLE_TFT='{TTP:1,UCE:1,ARF:1}']!")
+        if "ARF:1" in tft_env:
+            logger.warning(f"Disable hccl watchdog when using ARF.")
+            context.set_context(ascend_config={"hccl_watchdog": False})
+            logger.warning(f"Turn on TTP config when using ARF.")
+            if "TTP:1,UCE:1,ARF:1" not in tft_env:
+                os.environ["MS_ENABLE_TFT"] = "{TTP:1,ARF:1}"
+            os.environ["MS_ENABLE_RECOVERY"] = "1"
+
+        mode = context.get_context("mode")
+        device_target = context.get_context("device_target")
+        if device_target != "Ascend" or mode != context.GRAPH_MODE:
+            logger.warning(f"MindIO adataper only support on Ascend device with GRAPH Mode!"
+                           f"device:{device_target}, run mode: {mode}")
+            return
+
+        ctrl_port = int(os.getenv("MS_TFT_PORT"))
+        ctrl_ip = os.getenv("MS_TFT_IP", "")
+        Validator.check_non_negative_int(ctrl_port)
+        self._controller_ip = ctrl_ip
+        self._controller_rank_id = 0
+        self._controller_port = ctrl_port
+        try:
+            from mindio_ttp import framework_ttp as tft
+            self.tft = tft
+        except BaseException as e:
+            raise ModuleNotFoundError(f"Module nopt found. Detail info {str(e)}")
+        world_size = int(os.getenv("MS_WORKER_NUM"))  # from msrun
+        cur_rank = int(os.getenv("MS_NODE_ID"))  # from msrun
+        enable_local_copy = False
+        enable_arf = True if "ARF:1" in tft_env else False  # pylint: disable=simplifiable-if-expression
+        enable_tls = False
+        tls_key_dir = ""
+        self.enable_mindx = os.getenv("MINDX_TASK_ID")
+        # enable mindx, no need create controller
+        if cur_rank == self._controller_rank_id and self.enable_mindx is None:
+            logger.info(f"Begin to start tft controller on rank_id:{cur_rank}")
+            if enable_arf:
+                self._mindx_stub()
+            self.tft.tft_init_controller(cur_rank, world_size, enable_local_copy, enable_arf=enable_arf)
+            self.tft.tft_start_controller(self._controller_ip, self._controller_port, enable_tls, tls_key_dir)
+            logger.info("Finish start tft controller.")
+
+        logger.info("Begin to start tft processor.")
+        _tft_start_record_threads()
+        self.tft.tft_init_processor(cur_rank, world_size, enable_local_copy, enable_tls, tls_key_dir,
+                                    enable_arf=enable_arf)
+        self.tft.tft_start_processor(self._controller_ip, self._controller_port)
+        _tft_finish_record_threads()
+        logger.info("Finished start tft processor.")
+        if self.tft.tft_is_reboot_node():
+            logger.warning("tft report reboot init finish ")
+            tft.tft_report_error(tft.ReportState.RS_INIT_FINISH.value)
+            _set_recovery_context(is_reboot_node=True)
+            if config is None:
+                raise ValueError(
+                    f"Param 'config' can not be None when using ARF, and 'config' should be a dict type.")
+            logger.warning("tft reboot node no need load checkpoint when using ARF.")
+            config["arf_skip_load"] = True
+            ret = tft.tft_wait_next_action()
+            if ret != tft.Action.RETRY.value:
+                raise RuntimeError(f"ARF init failed!")
+
+
+_tft_handler = TftHandle()

mindspore/version.py

@@ -1 +1 @@
-__version__ = '2.5.0'
+__version__ = '2.6.0'

{mindspore-2.5.0.dist-info → mindspore-2.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mindspore
-Version: 2.5.0
+Version: 2.6.0
 Summary: MindSpore is a new open source deep learning training/inference framework that could be used for mobile, edge and cloud scenarios.
 Home-page: https://www.mindspore.cn
 Download-URL: https://github.com/mindspore-ai/mindspore/tags
@@ -321,6 +321,7 @@ Project stable branches will be in one of the following states:
 
 | **Version** | **Status**   | **Initial Release Date** | **Next Phase**                         | **EOL Date**|
 |------------|--------------|--------------------------|----------------------------------------|-------------|
+| **r2.5**   | Maintained   | 2025-02-08           |Unmaintained <br> 2026-02-08 estimated | 2026-02-08 |
 | **r2.4**   | Maintained   | 2024-10-30               | Unmaintained <br> 2025-10-30 estimated | 2025-10-30  |
 | **r2.3**   | Maintained   | 2024-07-15               | Unmaintained <br> 2025-07-15 estimated | 2025-07-15  |
 | **r2.2**   | End Of Life  | 2023-10-18               |                                        | 2024-10-18  |