mindspore 2.6.0__cp310-cp310-manylinux1_x86_64.whl → 2.7.0rc1__cp310-cp310-manylinux1_x86_64.whl

mindspore/utils/runtime_execution_order_check.py

@@ -22,11 +22,86 @@ from multiprocessing import cpu_count
 from typing import List, Dict, Union, Optional
 import sys
 import mindspore.log as logger
+from mindspore._c_expression import CommExecOrderChecker
 
 # Set Recursion Depth Limit
 sys.setrecursionlimit(10000)
 # support hccl group 150000 card
-csv.field_size_limit(1024 * 1024)
+csv.field_size_limit(1024 * 1024 * 10)
+
+
+def comm_exec_order_check(action):
+    """
+    Call the CommExecOrderCheck class to start the collection of communication operator execution sequences
+    or stop the collection and validate the execution order.
+
+    Args:
+        action (str): Control command - 'start' to begin collection, 'end' to stop and validate.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> import mindspore as ms
+        >>> from mindspore.utils import comm_exec_order_check
+        >>> comm_exec_order_check("start")
+        >>> model.train(1, train_dataset)
+        >>> comm_exec_order_check("end")
+    """
+    if not isinstance(action, str):
+        raise TypeError("The 'action' parameter must be a string.")
+    checker = CommExecOrderCheck()
+    checker(action)
+
+
+class CommExecOrderCheck:
+    """Controller for communication execution order verification.
+
+    Provides interface for starting/stopping the collection of communication
+    operator execution sequences. Integrates with C++ backend for actual
+    order tracking.
+    """
+    _instance = None
+
+    def __new__(cls, *args, **kwargs):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+        return cls._instance
+
+    def __init__(self):
+        if not hasattr(self, 'initialized'):
+            self.action = None
+            self.order_checker = CommExecOrderChecker.get_instance()
+            self.is_collecting = False
+            self.initialized = True
+
+    def __call__(self, action):
+        """
+        Args:
+            action (str): Control command - 'start' to begin collection,
+                        'end' to stop and validate
+        """
+        self.action = action
+        if action == "start":
+            self.start_function()
+        elif action == "end":
+            self.end_function()
+        else:
+            raise ValueError("Invalid action. Please use 'start' or 'end'.")
+
+    def start_function(self):
+        if self.is_collecting:
+            logger.error("The 'start' action cannot be called twice.")
+            return
+        self.is_collecting = True
+        self.order_checker.start_collect_exec_order()
+
+    def end_function(self):
+        if not self.is_collecting:
+            logger.error("The 'end' action cannot be called before the 'start' action.")
+            return
+        self.is_collecting = False
+        self.order_checker.stop_collect_exec_order()
 
 
 class ExecuteOrder:
@@ -54,12 +129,12 @@ class ExecuteOrder:
             comm_str = ",".join(self.comm_rank)
             return f"{self.primitive}_{self.group}_({comm_str})"
 
-        if self.primitive == "Send":
+        if self.primitive in ["Send", "DistCommIsend", "InnerCommIsend"]:
             # Unique base key of the Send operation.
             return f"Send_Receive_{self.group}_({rank})->({self.dest_rank})_{self.input_shape}"
 
-        if self.primitive == "Receive":
-            # Unique base key of the reception operation
+        if self.primitive in ["Receive", "DistCommIrecv", "InnerCommIrecv"]:
+            # Unique base key of the Recv operation
             return f"Send_Receive_{self.group}_({self.src_rank})->({rank})_{self.output_shape}"
 
         # Other operations, such as broadCast
@@ -202,10 +277,14 @@ class RankFolderParser:
             rank_id = os.path.basename(path).split("_")[1]
             # Adding one more layer to access the "execute_order" folder
             execute_order_path = os.path.join(path, "execute_order")
-            if os.path.exists(execute_order_path):
-                rank_result = self.parse_rank_folder(execute_order_path, rank_id)
-                if rank_result:
-                    result[rank_id] = rank_result[1]  # Extract execute orders
+            if not os.path.exists(execute_order_path):
+                raise FileNotFoundError(
+                    f"Execute order folder does not exist: {execute_order_path} "
+                    f"for rank_{rank_id} folder."
+                )
+            rank_result = self.parse_rank_folder(execute_order_path, rank_id)
+            if rank_result:
+                result[rank_id] = rank_result[1]  # Extract execute orders
             return result
 
         # If the path is a directory containing rank_{x} folders, parse all
@@ -217,8 +296,12 @@ class RankFolderParser:
                     rank_folder_path = os.path.join(path, d)
                     execute_order_path = os.path.join(rank_folder_path, "execute_order")
 
-                    if os.path.exists(execute_order_path):
-                        futures.append(thread_executor.submit(self.parse_rank_folder, execute_order_path, rank_id))
+                    if not os.path.exists(execute_order_path):
+                        raise FileNotFoundError(
+                            f"Execute order folder does not exist: {execute_order_path} "
+                            f"for rank_{rank_id} folder."
+                        )
+                    futures.append(thread_executor.submit(self.parse_rank_folder, execute_order_path, rank_id))
 
             for future in as_completed(futures):
                 try:
@@ -285,23 +368,22 @@ class RankFolderParser:
 
 def modify_execute_orders(execute_orders_map: dict) -> dict:
     """
-     Modify and generate unique execution order keys for each rank.
+    Modify and generate unique execution order keys for each rank.
 
-     This function processes a mapping of execution orders grouped by ranks. For each order,
-     it generates a unique key by combining a base key and a counter, ensuring all orders
-     are uniquely identifiable. The result is a dictionary where the keys are rank identifiers
-     and the values are lists of unique execution order keys.
+    This function processes a mapping of execution orders grouped by ranks. For each order,
+    it generates a unique key by combining a base key and a counter, ensuring all orders
+    are uniquely identifiable. The result is a dictionary where the keys are rank identifiers
+    and the values are lists of unique execution order keys.
 
-     Args:
-         execute_orders_map (dict): A dictionary where keys are rank identifiers (e.g., "rank_0")
-                                    and values are lists of ExecuteOrder objects. If a rank has no
-                                    orders, its value may be `None`.
-
-     Returns:
-         dict: A dictionary where keys are rank identifiers and values are lists of unique string
-               keys representing the modified execution orders for each rank.
+    Args:
+        execute_orders_map (dict): A dictionary where keys are rank identifiers (e.g., "rank_0")
+                                and values are lists of ExecuteOrder objects. If a rank has no
+                                orders, its value may be `None`.
 
-     """
+    Returns:
+        dict: A dictionary where keys are rank identifiers and values are lists of unique string
+            keys representing the modified execution orders for each rank.
+    """
     result = {}
 
     for rank, execute_orders in execute_orders_map.items():
@@ -331,26 +413,25 @@ def modify_execute_orders(execute_orders_map: dict) -> dict:
 
 def parse_and_validate(data: dict, all_rank: bool = True):
     """
-     Parse and validate execution orders in a directed graph structure.
+    Parse and validate execution orders in a directed graph structure.
 
-     This function checks the integrity and consistency of a given dataset, ensuring all required
-     keys are present and correctly referenced. It also validates the structure of the input data
-     and parses string values to extract meaningful components.
+    This function checks the integrity and consistency of a given dataset, ensuring all required
+    keys are present and correctly referenced. It also validates the structure of the input data
+    and parses string values to extract meaningful components.
 
-     Args:
-         data (dict): A dictionary where keys are string identifiers and values are lists of strings.
-                      Each value represents a dependency or reference to other keys.
-         all_rank (bool): If True, checks that all elements referenced in the data are present as keys
-                          in the dictionary. If False, only checks intersections.
-
-     Returns:
-         None: Log error messages to the console if validation fails, otherwise completes silently.
+    Args:
+        data (dict): A dictionary where keys are string identifiers and values are lists of strings.
+                    Each value represents a dependency or reference to other keys.
+        all_rank (bool): If True, checks that all elements referenced in the data are present as keys
+                        in the dictionary. If False, only checks intersections.
 
-     Raises:
-         ValueError: Raised indirectly if `parse_elements` encounters malformed input strings.
-         TypeError: Raised indirectly if data contains unexpected types.
+    Returns:
+        None: Log error messages to the console if validation fails, otherwise completes silently.
 
-     """
+    Raises:
+        ValueError: Raised indirectly if `parse_elements` encounters malformed input strings.
+        TypeError: Raised indirectly if data contains unexpected types.
+    """
     def parse_elements(value: str, max_groups: int = 2) -> set:
         """Extract unique elements inside the first one or two parentheses from a string."""
         groups = re.findall(r'\((.*?)\)', value)
@@ -398,9 +479,8 @@ def detect_cycle_in_graph(ranks_map):
 
     Returns:
     - tuple: (cycle_path, cycle_ranks) where cycle_path is a list of nodes forming the cycle and cycle_ranks
-             is a list of rank transitions corresponding to the cycle path.
+                is a list of rank transitions corresponding to the cycle path.
     """
-    # Step 1: Build the directed graph and track edges with ranks
     graph = defaultdict(list)
     rank_edges = {}
 
@@ -410,46 +490,50 @@ def detect_cycle_in_graph(ranks_map):
             graph[u].append(v)
             rank_edges[(u, v)] = rank
 
-    # Step 2: Detect cycle using DFS with path and rank tracking
     visited = set()
-    recursion_stack = set()
     path = []
+    node_indices = {}
     cycle_path = []
     cycle_ranks = []
 
-    def dfs(node):
-        if node in recursion_stack:  # Cycle detected
-            # Identify the cycle path from the recursion stack
-            cycle_index = path.index(node)
-            cycle_path.extend(path[cycle_index:])
-            for i in range(cycle_index, len(path) - 1):
-                u, v = path[i], path[i + 1]
-                cycle_ranks.append(f"{rank_edges[(u, v)]} {u} -> {v}")
-            # Add the closing edge for the cycle
-            cycle_ranks.append(f"{rank_edges[(path[-1], node)]} {path[-1]} -> {node}")
-            return True
+    # Use a stack to simulate recursion
+    stack = []
+    for node in list(graph.keys()):
+        if node not in visited:
+            stack.append((node, False))  # (node, is_processed)
 
-        if node in visited:
-            return False
+            while stack:
+                current_node, is_processed = stack.pop()
 
-        visited.add(node)
-        recursion_stack.add(node)
-        path.append(node)
+                if is_processed:
+                    # Post-processing: remove node from path and indices
+                    path.pop()
+                    del node_indices[current_node]
+                    continue
+
+                if current_node in node_indices:
+                    # Found a cycle
+                    cycle_start = node_indices[current_node]
+                    cycle_path = path[cycle_start:] + [current_node]
+                    for i in range(cycle_start, len(path)):
+                        u = path[i]
+                        v = path[i + 1] if i + 1 < len(path) else current_node
+                        cycle_ranks.append(f"{rank_edges[(u, v)]} {u} -> {v}")
+                    return cycle_path, cycle_ranks
+
+                if current_node in visited:
+                    continue
 
-        for neighbor in graph[node]:
-            if dfs(neighbor):
-                return True
+                visited.add(current_node)
+                node_indices[current_node] = len(path)
+                path.append(current_node)
 
-        # Backtrack
-        recursion_stack.remove(node)
-        path.pop()
-        return False
+                # Mark current node as processed
+                stack.append((current_node, True))
 
-    nodes = list(graph.keys())
-    for node in nodes:
-        if node not in visited:
-            if dfs(node):
-                return cycle_path, cycle_ranks
+                # Add neighbors to stack
+                for neighbor in reversed(graph[current_node]):
+                    stack.append((neighbor, False))
 
     return None, None
 
@@ -487,13 +571,13 @@ def output_cycle_results(cycle_path, cycle_ranks):
         None: Outputs results to the console.
     """
     if cycle_path:
-        logger.warning("Cycle detected:")
-        logger.warning(" -> ".join(cycle_path) + f" -> {cycle_path[0]}")  # Close the cycle
-        logger.warning("Involving ranks:")
+        logger.error("Cycle detected:")
+        logger.error(" -> ".join(cycle_path) + f" -> {cycle_path[0]}")  # Close the cycle
+        logger.error("Involving ranks:")
         for rank in cycle_ranks:
-            logger.warning(rank)
+            logger.error(rank)
     else:
-        logger.warning("Check success.")
+        logger.warning("Cycle Check success. There is no cycle in the graph.")
 
 
 def runtime_execution_order_check(folders_, all_rank=None):
@@ -531,7 +615,7 @@ def runtime_execution_order_check(folders_, all_rank=None):
     if all_rank is None:
         all_rank = determine_all_rank(folders_)
 
-    if all_rank is None:  # Input validation failed
+    if folders_ is None:  # Input validation failed
         logger.error("Invalid input. `folders_` must be a non-empty string or a list with at least one string element.")
         return
 

mindspore/utils/sdc_detect.py

@@ -0,0 +1,68 @@
+# Copyright 2025 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""SDC detect."""
+from mindspore import _c_expression
+
+
+def sdc_detect_start():
+    """
+    Start silent data corruption detection. It will check the inputs and outputs of MatMul operations during the
+    forward and backward computations on the current device, which may increase execution time. The overhead of the
+    check time decreases as the matrix shapes increase. Starting sdc detection results in approximately 100%
+    performance degradation for a single 4096-sized MatMul computation, and approximately 90% degradation on the
+    Llama2-7B model (model parallel is 4, pipeline parallel is 2, and using qkv concatenation and ffn concatenation in
+    decoder layers).
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> from mindspore.utils import sdc_detect_start
+        >>> sdc_detect_start()
+    """
+    _c_expression.sdc_detect_start()
+
+
+def sdc_detect_stop():
+    """
+    Stop silent data corruption detection.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> from mindspore.utils import sdc_detect_stop
+        >>> sdc_detect_stop()
+    """
+    _c_expression.sdc_detect_stop()
+
+
+def get_sdc_detect_result():
+    """
+    Get the result of silent data corruption detection.
+
+    Returns:
+        bool, indicating whether silent data corruption has occurred after detection start.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        >>> from mindspore.utils import get_sdc_detect_result
+        >>> result = get_sdc_detect_result()
+        >>> print(result)
+        False
+    """
+    return _c_expression.get_sdc_detect_result()

mindspore/utils/utils.py

@@ -117,17 +117,18 @@ class TftHandle:
         tft_register_mindx_callback('report_result', report_result)
         logger.warning('Stub register mindx func success.')
 
-    def init(self, config=None, **kwargs):
+    def init(self, **kwargs):
         """
         TFT handle init fun. Mainly used to initialize the mindio component.
 
         Args:
-            config (Dict): configs or the training model running. If ARF is not used, this parameter can be None.
             **kwargs: Reserved parameters.
         """
         tft_env = os.getenv("MS_ENABLE_TFT", "")
-        if ("TTP:1" not in tft_env) and ("UCE:1" not in tft_env) and ("ARF:1" not in tft_env):
-            raise ValueError("MindIO TFT regitster need custom switch on[MS_ENABLE_TFT='{TTP:1,UCE:1,ARF:1}']!")
+        tft_opts = ["TTP:1", "UCE:1", "HCCE:1", "ARF:1", "TSP:1"]
+        tft_enabled = any([opt in tft_env for opt in tft_opts])
+        if not tft_enabled:
+            raise ValueError("MindIO TFT regitster need custom switch on[MS_ENABLE_TFT='{%s}']!" % ",".join(tft_opts))
         if "ARF:1" in tft_env:
             logger.warning(f"Disable hccl watchdog when using ARF.")
             context.set_context(ascend_config={"hccl_watchdog": False})
@@ -181,14 +182,10 @@ class TftHandle:
             logger.warning("tft report reboot init finish ")
             tft.tft_report_error(tft.ReportState.RS_INIT_FINISH.value)
             _set_recovery_context(is_reboot_node=True)
-            if config is None:
-                raise ValueError(
-                    f"Param 'config' can not be None when using ARF, and 'config' should be a dict type.")
-            logger.warning("tft reboot node no need load checkpoint when using ARF.")
-            config["arf_skip_load"] = True
             ret = tft.tft_wait_next_action()
             if ret != tft.Action.RETRY.value:
                 raise RuntimeError(f"ARF init failed!")
+            logger.warning("tft reboot success.")
 
 
 _tft_handler = TftHandle()

mindspore/version.py

@@ -1 +1 @@
-__version__ = '2.6.0'
+__version__ = '2.7.0rc1'

{mindspore-2.6.0.dist-info → mindspore-2.7.0rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mindspore
-Version: 2.6.0
+Version: 2.7.0rc1
 Summary: MindSpore is a new open source deep learning training/inference framework that could be used for mobile, edge and cloud scenarios.
 Home-page: https://www.mindspore.cn
 Download-URL: https://github.com/mindspore-ai/mindspore/tags
@@ -332,11 +332,12 @@ Project stable branches will be in one of the following states:
 
 ## Maintenance status
 
-| **Version** | **Status**   | **Initial Release Date** | **Next Phase**                         | **EOL Date**|
+| **Version**| **Status**  | **Initial Release Date**| **Next Phase**                        | **EOL Date**|
 |------------|--------------|--------------------------|----------------------------------------|-------------|
-| **r2.5**   | Maintained   | 2025-02-08           |Unmaintained <br> 2026-02-08 estimated | 2026-02-08 |
+| **r2.6**   | Maintained   | 2025-05-19               | Unmaintained <br> 2026-05-19 estimated | 2026-05-19  |
+| **r2.5**   | Maintained   | 2025-02-08               | Unmaintained <br> 2026-02-08 estimated | 2026-02-08  |
 | **r2.4**   | Maintained   | 2024-10-30               | Unmaintained <br> 2025-10-30 estimated | 2025-10-30  |
-| **r2.3**   | Maintained   | 2024-07-15               | Unmaintained <br> 2025-07-15 estimated | 2025-07-15  |
+| **r2.3**   | End Of Life  | 2024-07-15               |                                        | 2025-07-15  |
 | **r2.2**   | End Of Life  | 2023-10-18               |                                        | 2024-10-18  |
 | **r2.1**   | End Of Life  | 2023-07-29               |                                        | 2024-07-29  |
 | **r2.0**   | End Of Life  | 2023-06-15               |                                        | 2024-06-15  |