mindspore 2.6.0__cp311-cp311-win_amd64.whl → 2.7.0rc1__cp311-cp311-win_amd64.whl

mindspore/parallel/cluster/process_entity/_utils.py

@@ -18,6 +18,8 @@ import json
 import socket
 import ipaddress
 import mindspore.log as logger
+from mindspore.runtime.thread_bind_core import _get_physical_device_id, _get_cpu_available, \
+    _auto_generate_strategy, _equal_distribution_strategy
 
 CURRENT_IP = None
 
@@ -45,19 +47,19 @@ def _generate_cmd_args_list(cmd, cmd_args):
     return [cmd] + cmd_args
 
 
-def _generate_cmd_args_list_with_core(cmd, cmd_args, cpu_start, cpu_end):
+def _generate_cmd_args_list_with_core(cmd, cmd_args, affinity_cpu_str):
     """
     Generates arguments list for 'Popen'. It consists of a binary file name and subsequential arguments.
     """
     # Bind cpu cores to this process.
-    taskset_args = ['taskset'] + ['-c'] + [str(cpu_start) + '-' + str(cpu_end)]
+    taskset_args = ['taskset'] + ['-c'] + [affinity_cpu_str]
     final_cmd = []
     if cmd not in ['python', 'pytest', 'python3']:
         # If user don't set binary file name, defaulty use 'python' to launch the job.
         final_cmd = taskset_args + ['python'] + [cmd] + cmd_args
     else:
         final_cmd = taskset_args + [cmd] + cmd_args
-    logger.info(f"Launch process with command: {' '.join(final_cmd)}")
+    logger.warning(f"Launch process with command: {' '.join(final_cmd)}")
     return final_cmd
 
 
@@ -83,8 +85,8 @@ def _get_local_ip(ip_address):
             CURRENT_IP = s.getsockname()[0]
             s.close()
         except Exception as e:
-            raise RuntimeError(f"Get local ip failed: {e}. Please check whether an accessible address "
-                               "is input by '--master_address'.")
+            raise RuntimeError("Get local ip has failed. Please verify that the accessible address has been "
+                               "specified in the '--master_address' parameter") from e
     return CURRENT_IP
 
 
@@ -124,8 +126,8 @@ def _convert_addr_to_ip(master_addr):
             logger.info(f"Convert input host name:{master_addr} to ip address:{ip_address}.")
             return ip_address
         except socket.gaierror as e:
-            raise RuntimeError(f"DNS resolution failed: {e}. Please check whether a correct host name "
-                               "is input by '--master_address'.")
+            raise RuntimeError("DNS resolution has failed. Please verify that the correct hostname has been "
+                               "specified in the '--master_address' parameter") from e
 
 
 def _send_scale_num(url, scale_num):
@@ -134,3 +136,89 @@ def _send_scale_num(url, scale_num):
 
     """
     return ""
+
+
+def _parse_global_device_to_cpu_map(local_rank_id, physical_device_id, device_to_cpu_map):
+    """
+    Parse the global device_to_cpu_map and return a cpu list for assigned local_rank_id.
+
+    """
+    input_device_id = int(list(device_to_cpu_map.keys())[local_rank_id].replace("device", ""))
+    if physical_device_id != input_device_id:
+        return ""
+    affinity_cpu_list = list(device_to_cpu_map.values())[local_rank_id]
+    affinity_cpu_str = ",".join(affinity_cpu_list)
+    return affinity_cpu_str
+
+
+def _generate_auto_bind_core_strategy(local_worker_num):
+    """
+    Get device to core range assigned for the all processes.
+
+    """
+    simulation_level = os.getenv("MS_SIMULATION_LEVEL", "").strip()
+
+    try:
+        available_cpus = _get_cpu_available()
+    except RuntimeError as e:
+        logger.warning(f"Failed to acquire available cpu info, error: {e} Will not launch process with taskset.")
+        return {}
+
+    if not simulation_level:
+        device_to_cpu_map = _auto_generate_strategy(local_worker_num, available_cpus)
+    else:
+        device_to_cpu_map = _equal_distribution_strategy(local_worker_num, available_cpus)
+
+    return device_to_cpu_map
+
+
+def ranges_to_str(num_list):
+    """
+    Convert a num list to a range string.
+
+    """
+    ranges = []
+    start = num_list[0]
+    for i in range(1, len(num_list)):
+        if num_list[i] != num_list[i-1] + 1:
+            ranges.append((start, num_list[i-1]))
+            start = num_list[i]
+    ranges.append((start, num_list[-1]))
+
+    parts = []
+    for start, end in ranges:
+        if start == end:
+            parts.append(str(start))
+        else:
+            parts.append(f"{start}-{end}")
+    return ",".join(parts)
+
+
+def _generate_bind_core_strategy(local_rank_id, device_to_cpu_map, arg_bind_core):
+    """
+    Get device to core range assigned for the all processes.
+
+    """
+    affinity_cpu_str = ""
+    cpu_list_for_device = []
+    simulation_level = os.getenv("MS_SIMULATION_LEVEL", "").strip()
+
+    try:
+        physical_device_id = _get_physical_device_id(local_rank_id, simulation_level)
+    except RuntimeError as e:
+        logger.warning(f"Failed to acquire device id, error: {e} Will not launch process with taskset.")
+        return None
+
+    if isinstance(arg_bind_core, dict):
+        affinity_cpu_str = _parse_global_device_to_cpu_map(local_rank_id, physical_device_id, arg_bind_core)
+        if not affinity_cpu_str:
+            logger.warning(f"Failed to find physical_device_id[{physical_device_id}] for "
+                           f"process[{local_rank_id}]. Will not launch process with taskset.")
+            return None
+    elif arg_bind_core is True:
+        cpu_list_for_device = device_to_cpu_map.get(physical_device_id, [])
+        if not cpu_list_for_device:
+            return None
+        os.environ["MSRUN_CPU_LIST"] = str(cpu_list_for_device)
+        affinity_cpu_str = ranges_to_str(cpu_list_for_device)
+    return affinity_cpu_str

mindspore/parallel/cluster/run.py

@@ -14,9 +14,47 @@
 # ============================================================================
 """Entrypoint of ms_run"""
 import ast
-from argparse import REMAINDER, ArgumentParser
+import re
+import json
+from argparse import REMAINDER, ArgumentParser, ArgumentTypeError
 from .process_entity import _ProcessManager
 
+
+def parse_and_validate_bind_core(value):
+    """
+    Parse input argument of --bind_core.
+
+    """
+    if value.lower() == "true":
+        return True
+    if value.lower() == "false":
+        return False
+
+    try:
+        value_dict = json.loads(value)
+    except json.JSONDecodeError as e:
+        raise ArgumentTypeError("Failed to parse JSON into a dictionary") from e
+
+    if isinstance(value_dict, dict):
+        range_pattern = re.compile(r'^\d+-\d+$')
+        for device_id, affinity_cpu_list in value_dict.items():
+            if not re.fullmatch(r"device\d+", device_id):
+                raise ArgumentTypeError(f"Key '{device_id}' must be in format 'deviceX' (X ≥ 0).")
+            if not isinstance(affinity_cpu_list, list):
+                raise ArgumentTypeError(f"Value for '{device_id}':{affinity_cpu_list} should be a list, "
+                                        f"but got {type(affinity_cpu_list)}.")
+
+            for cpu_range in affinity_cpu_list:
+                if not isinstance(cpu_range, str):
+                    raise ArgumentTypeError(f"CPU range '{cpu_range}' in '{affinity_cpu_list}' should be a string.")
+                if not range_pattern.match(cpu_range):
+                    raise ArgumentTypeError(f"CPU range '{cpu_range}' in '{affinity_cpu_list}' should be "
+                                            "in format 'cpuidX-cpuidY'.")
+        return value_dict
+
+    raise ArgumentTypeError(f"Type of {value} should be bool or dict, but got {type(value)}.")
+
+
 def get_args():
     """
     Parses and retrieves command-line arguments.
@@ -77,9 +115,10 @@ def get_args():
     parser.add_argument(
         "--bind_core",
         default=False,
-        type=ast.literal_eval,
-        choices=[True, False],
-        help="specifies whether msrun should bind cpu cores to spawned processes."
+        type=parse_and_validate_bind_core,
+        help="specifies whether msrun should bind CPU cores to spawned processes. "
+             "If set to True, msrun will bind core based on the environment automatically, "
+             "and if passed a dict, msrun will bind core based on this dict information."
     )
     parser.add_argument(
         "--sim_level",

mindspore/parallel/function/__init__.py

@@ -16,8 +16,15 @@
 """
 Parallel function operator
 """
+from __future__ import absolute_import
 
-from mindspore.parallel.function.reshard_func import reshard
+from . import (
+    reshard_func
+)
+
+from .reshard_func import (
+    reshard
+)
 
 __all__ = []
 __all__.extend(reshard_func.__all__)

mindspore/parallel/function/reshard_func.py

@@ -235,7 +235,7 @@ def _redistribute(tensor, dst_dtensor_info):
     global REDIST_CELL_CACHE
     redist_cache_key = (f"{src_layout_info['device_matrix']}, {src_layout_info['tensor_map']} -> "
                         f"{dst_layout_info['device_matrix']}, {dst_layout_info['tensor_map']}")
-    if redist_cache_key in REDIST_CELL_CACHE.keys():
+    if redist_cache_key in REDIST_CELL_CACHE:
         logger.debug(f"redist_cache_key is {redist_cache_key}, match cache")
         redist_func = REDIST_CELL_CACHE[redist_cache_key]
     else:

mindspore/parallel/nn/__init__.py

@@ -17,8 +17,21 @@ Interfaces for parallel-related functionality
 """
 from __future__ import absolute_import
 
-from mindspore.parallel.nn.parallel_grad_reducer import PipelineGradReducer
-from mindspore.parallel.nn.parallel_cell_wrapper import PipelineCell, Pipeline, MicroBatchInterleaved, GradAccumulation
+from . import (
+    parallel_grad_reducer,
+    parallel_cell_wrapper
+)
+
+from .parallel_grad_reducer import (
+    PipelineGradReducer
+)
+
+from .parallel_cell_wrapper import (
+    PipelineCell,
+    Pipeline,
+    MicroBatchInterleaved,
+    GradAccumulation
+)
 
 __all__ = []
 __all__.extend(parallel_grad_reducer.__all__)

mindspore/parallel/nn/parallel_cell_wrapper.py

@@ -17,6 +17,8 @@
 from __future__ import absolute_import
 from __future__ import division
 
+__all__ = ['PipelineCell', 'Pipeline', 'MicroBatchInterleaved', 'GradAccumulation']
+
 from mindspore import nn
 from mindspore.ops import operations as P
 from mindspore.nn.cell import Cell
@@ -24,9 +26,6 @@ from mindspore.nn.wrap.cell_wrapper import _MicroBatch
 from mindspore import log as logger
 
 
-__all__ = ['PipelineCell', 'Pipeline', 'MicroBatchInterleaved', 'GradAccumulation']
-
-
 class PipelineCell(Cell):
     """
     Slice MiniBatch into finer-grained MicroBatch for use in pipeline-parallel training.
@@ -101,15 +100,15 @@ class PipelineCell(Cell):
                                    " config stage num:" + str(config_stage_num))
                 logger.warning("network:" + str(self.network))
                 logger.warning("cell name available:")
-                for cell_name, cell in self.network.cells_and_names():
+                for cell_name, _ in self.network.cells_and_names():
                     logger.warning(cell_name)
                 raise KeyError("For 'PipelineCell', the argument 'stage_config' : {} is not "
                                "found in 'network' : {}".format(config_dict, network))
 
-    def construct(self, *inputs):
+    def construct(self, *args, **kwargs):
         ret = None
         for i in range(self.micro_size):
-            micro_input = self.micro_inputs[i](i, *inputs)
+            micro_input = self.micro_inputs[i](i, *args, **kwargs)
             output = self.network(*micro_input)
             if ret is not None:
                 ret = self.add_list[i](ret, output)
@@ -197,10 +196,10 @@ class MicroBatchInterleaved(Cell):
             self.interleave_inputs.append(interleave_data)
         self._get_attr_from_cell(network)
 
-    def construct(self, *inputs):
+    def construct(self, *args, **kwargs):
         output = 0.0
         for i in range(self.interleave_num):
-            interleave_input = self.interleave_inputs[i](i, *inputs)
+            interleave_input = self.interleave_inputs[i](i, *args, **kwargs)
             output = self.add(output, self.network(*interleave_input))
         return output
 
@@ -251,10 +250,10 @@ class GradAccumulation(Cell):
             self.add_list.append(self.add)
         self._get_attr_from_cell(network)
 
-    def construct(self, *inputs):
+    def construct(self, *args, **kwargs):
         ret = None
         for i in range(self.micro_size):
-            micro_input = self.micro_inputs[i](i, *inputs)
+            micro_input = self.micro_inputs[i](i, *args, **kwargs)
             output = self.network(*micro_input)
             if ret is not None:
                 ret = self.add_list[i](ret, output)

mindspore/parallel/nn/parallel_grad_reducer.py

@@ -15,6 +15,8 @@
 """parallel serialization"""
 from __future__ import absolute_import
 
+__all__ = ['PipelineGradReducer']
+
 from mindspore import context
 from mindspore.nn.cell import Cell
 from mindspore.ops import functional as F, composite as C, operations as P
@@ -25,8 +27,6 @@ from mindspore.common.parameter import Parameter
 from mindspore.nn.layer import Identity
 from mindspore.parallel._utils import _get_enable_parallel_optimizer
 
-__all__ = ['PipelineGradReducer']
-
 
 grad_scale = C.MultitypeFuncGraph("grad_scale")
 shard_grad_scale = C.MultitypeFuncGraph("shard_grad_scale")
@@ -81,6 +81,7 @@ class PipelineGradReducer(Cell):
         >>> import mindspore as ms
         >>> from mindspore import nn, ops, Tensor
         >>> from mindspore.communication import init
+        >>> from mindspore.parallel.auto_parallel import AutoParallel
         >>>
         >>> ms.set_context(mode=ms.GRAPH_MODE)
         >>> ms.reset_auto_parallel_context()
@@ -113,7 +114,7 @@ class PipelineGradReducer(Cell):
         >>> net.layer3.pipeline_stage = 1
         >>> loss_fn = nn.CrossEntropyLoss()
         >>> optimizer = nn.SGD(net.trainable_params(), 1e-2)
-        >>> net_with_loss = nn.Pipeline(nn.WithLossCell(net, loss_fn), 2)
+        >>> net_with_loss = nn.PipelineCell(nn.WithLossCell(net, loss_fn), 2)
         >>> net_with_loss.set_train()
         >>> def forward_fn(inputs, target):
         ...     loss = net_with_loss(inputs, target)
@@ -135,7 +136,7 @@ class PipelineGradReducer(Cell):
         >>> label = Tensor(np.ones([size, out_features]).astype(np.float32))
         >>> loss, _ = train_one_step(inputs, label)
         >>> print(loss)
-        46.36721
+        46.304886
     """
     def __init__(self, parameters, scale_sense=1.0, opt_shard=None):
         super(PipelineGradReducer, self).__init__(auto_prefix=False)
@@ -151,10 +152,10 @@ class PipelineGradReducer(Cell):
             self.opt_shard = opt_shard
 
     @jit
-    def construct(self, grads):
+    def construct(self, *args, **kwargs):
         new_grads = None
         if self.opt_shard:
-            grads = self.grad_reducer(grads)
+            grads = self.grad_reducer(*args, **kwargs)
             new_grads = self.hyper_map(F.partial(shard_grad_scale, self.scale_sense * self.degree),
                                        grads, self.accu_grads)
         else:

mindspore/parallel/shard.py

@@ -120,8 +120,8 @@ class Layout:
         >>> layout = Layout((2, 2, 2), ("dp", "sp", "mp"))
         >>> layout0 = layout("dp", "mp")
         >>> print(layout0.to_dict())
-        {'device_matrix': (2, 2, 2), 'tensor_map': (2, 0), 'interleaved_parallel': False,
-        'alias_name': {'dp', 'sp', 'mp'}, 'rank_list': [0, 1, 2, 3, 4, 5, 6, 7]}
+        {"device_matrix": (2, 2, 2), "tensor_map": (2, 0), "interleaved_parallel": False,
+        'alias_name': {'dp', 'sp', 'mp'}, "rank_list": [0, 1, 2, 3, 4, 5, 6, 7]}
         >>> layout = Layout((2, 2, 2), ("dp", "sp", "interleaved_parallel"))
         >>> layout1 = layout(("dp", "interleaved_parallel"), "sp")
     """