mindspore 2.7.0__cp310-cp310-win_amd64.whl → 2.7.1__cp310-cp310-win_amd64.whl

mindspore/ops_generate/tensor_py_cc_generator.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 # ============================================================================
 """
-Generates mindspore/ccsrc/pybind_api/ir/tensor_py.cc which includes the CPython Tensor APIs.
+Generates mindspore/ccsrc/pybind_api/ir/tensor/tensor_py.cc which includes the CPython Tensor APIs.
 """
 
 import os
@@ -26,7 +26,7 @@ from pyboost import pyboost_utils
 
 class TensorPyCppGenerator(BaseGenerator):
     """
-    This class is responsible for generating mindspore/ccsrc/pybind_api/ir/tensor_register/
+    This class is responsible for generating mindspore/ccsrc/pybind_api/ir/tensor/tensor_register/
     auto_generate/tensor_py_gen.cc
     """
     def __init__(self):

mindspore/parallel/_cell_wrapper.py

@@ -263,7 +263,7 @@ def _single_parameter_broadcast(net, layout, param_not_load=None, param_loaded=N
     if not single_params:
         return
     param_redundancy_reversed = _get_param_redundancy_reversed(param_redundancy, cur_rank)
-    if not param_redundancy_reversed or cur_rank not in single_params:
+    if not param_redundancy_reversed:
         return
     net_param_dict = net.parameters_dict()
     _chang_parallel_context(origin_dataset_strategy)

mindspore/parallel/_parallel_serialization.py

@@ -526,10 +526,7 @@ def _make_dir(path, arg_name):
     else:
         ms.log.debug("The directory(%s) doesn't exist, will create it", path)
         try:
-            permissions = os.R_OK | os.W_OK | os.X_OK
-            os.umask(permissions << 3 | permissions)
-            mode = permissions << 6
-            os.makedirs(path, mode=mode, exist_ok=True)
+            os.makedirs(path, mode=0o700, exist_ok=True)
             real_path = path
         except PermissionError as e:
             ms.log.critical("No write permission on the directory(%r), error = %r", path, e)

mindspore/parallel/_utils.py

@@ -14,12 +14,13 @@
 # ============================================================================
 """Utils of auto parallel"""
 import os
+import re
 from time import perf_counter
 from importlib import import_module
 import numpy as np
 import mindspore as ms
 from mindspore import context, log as logger
-from mindspore._c_expression import reset_op_id, reset_op_id_with_offset
+from mindspore._c_expression import reset_op_id
 from mindspore.common.tensor import Tensor
 from mindspore.common.dtype import _dtype_to_nptype
 from mindspore.common import dtype as mstype
@@ -584,11 +585,6 @@ def _reset_op_id():
     reset_op_id()
 
 
-def _reset_op_id_with_offset():
-    """Reset op id with offset."""
-    reset_op_id_with_offset()
-
-
 def _parallel_predict_check():
     """validate parallel model prediction"""
     if _is_in_auto_parallel_mode():
@@ -798,3 +794,30 @@ def _check_rank(cur_rank, initial_rank, pipeline_stages):
         raise ValueError(f"For parameter broadcast, the cur_rank: {cur_rank} is wrong.")
     if initial_rank % (get_group_size() / pipeline_stages) != 0:
         raise ValueError(f"For parameter broadcast, the initial_rank: {initial_rank} is wrong.")
+
+
+def _check_path_safe(path, arg_name):
+    """
+    Check input path string is safe.
+    """
+    illegal_patterns = [
+        r"\.\.",
+        r"//+",
+        r"~",
+        r"^\s*$",
+        r"\./\."
+    ]
+    for pattern in illegal_patterns:
+        if re.search(pattern, path):
+            pattern_info = pattern.replace('\\', '')
+            raise ValueError(f"{arg_name} contains '{pattern_info}' is not safe, please use a safe one.")
+
+
+def _check_path_writable(path):
+    """
+    Check the write permission of the input path.
+    """
+    if not os.path.exists(path):
+        raise RuntimeError(f"{path} Path does not exist.")
+    if not os.access(path, os.W_OK):
+        raise PermissionError(f"Don't have the write permission on the directory {path}.")

mindspore/parallel/checkpoint_transform.py

@@ -31,7 +31,7 @@ from mindspore.communication.management import get_rank, get_group_size
 from mindspore.parallel._tensor import _load_tensor, _reshape_param_data, _reshape_param_data_with_weight, \
     _get_tensor_slice_index, _get_tensor_strategy
 from mindspore.parallel._utils import _is_in_auto_parallel_mode, _get_pipeline_stages, _infer_rank_list, \
-    _remove_repeated_slices, _get_auto_parallel_net
+    _remove_repeated_slices, _get_auto_parallel_net, _check_path_safe, _check_path_writable
 from mindspore.parallel._parallel_serialization import _rank_list_for_transform_parallel_checkpoint, \
     _transform_parallel_checkpoint, _get_device_num_from_strategy, _make_dir, _build_searched_strategy, \
     _extract_layout_map, _extract_src_dst_layout_map, _parameter_not_in_local_stage, _extract_pipeline_stage_num, \
@@ -69,7 +69,9 @@ def merge_pipeline_strategys(src_strategy_dirs, dst_strategy_file):
         >>> ms.parallel.merge_pipeline_strategys("./src_strategy_dir", "./dst_strategy.ckpt")
 
     """
-    dst_strategy_dir, _ = os.path.split(dst_strategy_file)
+    dst_strategy_file = os.path.normpath(dst_strategy_file)
+    dst_strategy_file = os.path.abspath(dst_strategy_file)
+    dst_strategy_dir = os.path.dirname(dst_strategy_file)
     if not os.path.exists(dst_strategy_dir):
         _make_dir(dst_strategy_dir, "path")
     if not os.path.isdir(src_strategy_dirs):
@@ -495,6 +497,9 @@ def _transform_checkpoint_by_stage(src_checkpoints_dir, dst_checkpoints_dir, ckp
 def _transform_checkpoints(src_checkpoints_dir, dst_checkpoints_dir, ckpt_prefix, src_strategy_file=None,
                            dst_strategy_file=None):
     """Transform checkpoints for all stages in src_strategy_file"""
+    _check_path_safe(dst_checkpoints_dir, "dst_checkpoints_dir")
+    dst_checkpoints_dir = os.path.realpath(dst_checkpoints_dir)
+    _check_path_safe(ckpt_prefix, "ckpt_prefix")
     checkpoints_rank_dir_list = os.path.join(src_checkpoints_dir, "rank_[0-9]*")
     all_checkpoint_files_map = {}
     for checkpoint_dir in glob.glob(checkpoints_rank_dir_list):
@@ -563,6 +568,7 @@ def _transform_checkpoints(src_checkpoints_dir, dst_checkpoints_dir, ckpt_prefix
             save_checkpoint_file_dir = os.path.join(dst_checkpoints_dir, "rank_{}".format(transform_rank))
             if not os.path.exists(save_checkpoint_file_dir):
                 _make_dir(save_checkpoint_file_dir, "path")
+            _check_path_writable(save_checkpoint_file_dir)
             save_checkpoint_file_name = os.path.join(save_checkpoint_file_dir, save_checkpoint_file)
             ms.save_checkpoint(transform_param_list, save_checkpoint_file_name)
             del param_total_dict_copy
@@ -913,6 +919,15 @@ def set_op_strategy_config(mode="SAVE", path=""):
     if file_type != ".json":
         raise KeyError("File type must be .json")
     dir_path = os.path.dirname(path)
+
+    normalized_path = os.path.abspath(os.path.realpath(path))
+    dangerous_paths = ['/etc', '/usr', '/bin', '/sbin', '/boot', '/proc', '/sys']
+    for dangerous_path in dangerous_paths:
+        if normalized_path.startswith(dangerous_path):
+            raise PermissionError(
+                f"Writing to system directory '{dangerous_path}' is not allowed"
+            )
+
     if dir_path and not os.path.exists(dir_path):
         os.makedirs(dir_path, mode=0o700, exist_ok=True)
     check_mode_type = ["SAVE", "LOAD"]
@@ -1182,6 +1197,7 @@ def load_distributed_checkpoint(network, checkpoint_filenames=None, predict_stra
 
     param_total_dict = defaultdict(dict)
     for file_index, file_name in enumerate(checkpoint_filenames):
+        file_name = os.path.abspath(file_name)
         ckpt_dict = ms.load_checkpoint(file_name, dec_key=dec_key, dec_mode=dec_mode)
         for param_name, param in ckpt_dict.items():
             param_total_dict[param_name][file_index] = param

mindspore/parallel/cluster/process_entity/_api.py

@@ -21,6 +21,7 @@ import subprocess
 import socket
 import psutil
 import mindspore.log as logger
+from mindspore.utils import RSCPluginHandle
 from ._utils import _generate_cmd_args_list, _generate_cmd_args_list_with_core, _generate_url, \
     _is_local_ip, _convert_addr_to_ip, _send_scale_num, _get_local_ip, _generate_auto_bind_core_strategy, \
     _generate_bind_core_strategy
@@ -221,23 +222,28 @@ class _ProcessManager:
 
         self.proc_rank_map = {}
         self.enable_mindx = False
+        self.handler = None
         self._check_taskd()
 
     def _check_taskd(self):
         """check if enable taskd."""
-        tft_env = os.getenv("MS_ENABLE_TFT", "")
-        if any(v in tft_env for v in ('TTP:1', 'UCE:1', 'ARF:1', 'TSP:1', 'RSC:1', 'HCCE:1')):
-            try:
-                from taskd.python.framework.agent.ms_mgr.msrun_plugin import MSRunPlugin
-                self.msmgr = MSRunPlugin()
-                self.msmgr.register_callbacks("KILL_WORKER", self.kill_workers)
-                self.msmgr.register_callbacks("START_ALL_WORKER", self.start_all_workers)
-                self.msmgr.register_callbacks("START_WORKER_LIST", self.start_worker_list)
-                self.msmgr.register_callbacks("MONITOR", self.monitor_rank_status)
-                self.enable_mindx = True
-                os.environ["MS_ENABLE_RECOVERY"] = str(1)
-            except Exception as e:  # pylint: disable=broad-except
-                logger.warning(f"mindx is not installed, using original mindspore recovery strategy.: {str(e)}")
+        self.handler = RSCPluginHandle()
+        self.enable_mindx = self.handler.check_enable()
+        if self.enable_mindx is False:
+            self.handler = None
+            return
+        ret = self.handler.register_callback({"KILL_WORKER": self.kill_workers,
+                                              "START_ALL_WORKER": self.start_all_workers,
+                                              "START_WORKER_LIST": self.start_worker_list,
+                                              "MONITOR": self.monitor_rank_status
+                                              })
+        if not ret:
+            logger.warning(f"Register callback to mindx failed, process controlled by msrun.")
+            self.enable_mindx = False
+            self.handler = None
+            return
+        logger.warning(f"Mindx enabled, process controlled by mindx.")
+        os.environ["MS_ENABLE_RECOVERY"] = str(1)
 
     def run(self):
         """
@@ -260,7 +266,7 @@ class _ProcessManager:
             if self.is_master and not self.is_simulation:
                 self.start_scheduler()
         if self.enable_mindx:
-            self.msmgr.start()
+            self.handler.start()
         else:
             self.start_workers()
             if self.join:
@@ -382,8 +388,7 @@ class _ProcessManager:
                 logger.error(f"Scheduler process {self.msn_process.pid} exit with exception.")
 
         if has_exception:
-            logger.info("Analyzing exception log...")
-            self._analyze_log()
+            self._analyze_sched_log()
             raise RuntimeError("Distributed job exited with exception. Please check logs in "
                                f"directory: {self.log_dir}.")
 
@@ -583,26 +588,13 @@ class _ProcessManager:
             log_name = os.path.join(self.log_dir, formatted_log_name + "_" + str(index) + ".log")
         return node_id, log_name
 
-    def _analyze_log(self):
+    def _analyze_sched_log(self):
         """
-        Analyze exception logs.
+        Analyze scheduler log.
         """
         scheduler_log_path = os.path.join(self.log_dir, "scheduler.log")
-        time_out_node_ids = []
         if os.path.exists(scheduler_log_path):
-            with open(scheduler_log_path, "r") as log:
-                scheduler_log = log.read()
-                # Filter out abnormal logs.
-                time_out_node_log = re.findall(r"node: .* is timed out", scheduler_log)
-
-                # Filter out node ids of the processes which exit abnormally.
-                def node_id_splitter(node_id):
-                    return re.split(" is timed out", re.split("node: ", node_id)[1])[0]
-                for node_id in time_out_node_log:
-                    time_out_node_ids.append(node_id_splitter(node_id))
-            logger.error(f"Time out nodes are {time_out_node_ids}")
-
-        os.system(f"grep -rn -E 'ERROR|CRITICAL|Traceback|Error' -C 5 {self.log_dir}")
+            os.system(f"cat {scheduler_log_path} | grep -E 'ERROR|CRITICAL|Traceback|Error' -C 5")
 
     def format_worker_log_name(self):
         """

mindspore/parallel/cluster/process_entity/_utils.py

@@ -30,7 +30,7 @@ def _generate_cmd(cmd, cmd_args, output_name):
 
     """
     if cmd not in ['python', 'pytest', 'python3']:
-        # If user don't set binary file name, defaulty use 'python' to launch the job.
+        # If user don't set binary file name, defaultly use 'python' to launch the job.
         command = f"python {cmd} {' '.join(cmd_args)} > {output_name} 2>&1 &"
     else:
         command = f"{cmd} {' '.join(cmd_args)} > {output_name} 2>&1 &"
@@ -42,7 +42,7 @@ def _generate_cmd_args_list(cmd, cmd_args):
     Generates arguments list for 'Popen'. It consists of a binary file name and subsequential arguments.
     """
     if cmd not in ['python', 'pytest', 'python3']:
-        # If user don't set binary file name, defaulty use 'python' to launch the job.
+        # If user don't set binary file name, defaultly use 'python' to launch the job.
         return ['python'] + [cmd] + cmd_args
     return [cmd] + cmd_args
 
@@ -55,7 +55,7 @@ def _generate_cmd_args_list_with_core(cmd, cmd_args, affinity_cpu_str):
     taskset_args = ['taskset'] + ['-c'] + [affinity_cpu_str]
     final_cmd = []
     if cmd not in ['python', 'pytest', 'python3']:
-        # If user don't set binary file name, defaulty use 'python' to launch the job.
+        # If user don't set binary file name, defaultly use 'python' to launch the job.
         final_cmd = taskset_args + ['python'] + [cmd] + cmd_args
     else:
         final_cmd = taskset_args + [cmd] + cmd_args
@@ -143,8 +143,14 @@ def _parse_global_device_to_cpu_map(local_rank_id, physical_device_id, device_to
     Parse the global device_to_cpu_map and return a cpu list for assigned local_rank_id.
 
     """
+    if local_rank_id >= len(list(device_to_cpu_map.keys())):
+        logger.warning(f"Cannot find process[{local_rank_id}] in args '--bind_core'. "
+                       "Will not launch process with taskset.")
+        return ""
     input_device_id = int(list(device_to_cpu_map.keys())[local_rank_id].replace("device", ""))
     if physical_device_id != input_device_id:
+        logger.warning(f"Cannot find physical_device_id[{physical_device_id}] for process[{local_rank_id}] "
+                       "in args '--bind_core'. Will not launch process with taskset.")
         return ""
     affinity_cpu_list = list(device_to_cpu_map.values())[local_rank_id]
     affinity_cpu_str = ",".join(affinity_cpu_list)
@@ -212,8 +218,6 @@ def _generate_bind_core_strategy(local_rank_id, device_to_cpu_map, arg_bind_core
     if isinstance(arg_bind_core, dict):
         affinity_cpu_str = _parse_global_device_to_cpu_map(local_rank_id, physical_device_id, arg_bind_core)
         if not affinity_cpu_str:
-            logger.warning(f"Failed to find physical_device_id[{physical_device_id}] for "
-                           f"process[{local_rank_id}]. Will not launch process with taskset.")
             return None
     elif arg_bind_core is True:
         cpu_list_for_device = device_to_cpu_map.get(physical_device_id, [])

mindspore/{experimental/llm_boost/atb → parallel/distributed}/__init__.py

@@ -1,23 +1,21 @@
-# Copyright 2024 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""
-Provide llm boost for inference, such as LlamaBoost.
-"""
-from __future__ import absolute_import
-
-from mindspore.experimental.llm_boost.atb.llama_boost import LlamaBoost
-from mindspore.experimental.llm_boost.atb.qwen_boost import QwenBoost
-
-__all__ = ['LlamaBoost', 'QwenBoost']
+# Copyright 2025 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+"""distributed init"""
+from mindspore.parallel.distributed.distributed_data_parallel import DistributedDataParallel
+
+__all__ = [
+    "DistributedDataParallel",
+]