mindspore 2.6.0__cp310-cp310-manylinux1_x86_64.whl → 2.7.0rc1__cp310-cp310-manylinux1_x86_64.whl

mindspore/parallel/_tensor.py

@@ -21,9 +21,11 @@ import numpy as np
 from mindspore.common.tensor import Tensor
 from mindspore.communication.management import get_rank, get_group_size
 from mindspore._c_expression import TensorTransform
+from mindspore import log as logger
 
 _tensor_transform = TensorTransform.get_instance()
-
+COMM_TENSOR_CELL_CACHE = {}
+RESHARD_OP_MAP_CACHE = {}
 
 def _get_tensor_strategy(dev_mat, tensor_map):
     """
@@ -348,7 +350,7 @@ def _extract_layout_item(layout_item):
     return dev_matrix, tensor_map, opt_shard_step, opt_shard_size
 
 
-def _transform_tensor_by_layout(from_layout, to_layout, device_list, rank_id):
+def _transform_tensor_by_layout(from_layout, to_layout, device_list, rank_id, enable_redist_opt=False):
     """
     Transform tensor from source layout to the destination layout.
 
@@ -362,7 +364,7 @@ def _transform_tensor_by_layout(from_layout, to_layout, device_list, rank_id):
     """
     if not isinstance(from_layout, tuple) or not isinstance(to_layout, tuple):
         raise TypeError("The layout should be tuple! layout is {} and {}".format(from_layout, to_layout))
-    return _tensor_transform.transform_tensor_sharding(from_layout, to_layout, device_list, rank_id)
+    return _tensor_transform.transform_tensor_sharding(from_layout, to_layout, device_list, enable_redist_opt, rank_id)
 
 
 def _construct_from_to_tensor_layout(from_full_tensor_shape, from_dev_matrix,
@@ -587,13 +589,15 @@ def _get_needed_rank_list_by_layouts(from_tensor_layout, to_tensor_layout, devic
     return result_list
 
 
-def _get_needed_rank_transform_operator_map_by_layouts(from_tensor_layout, to_tensor_layout, device_list, self_rank):
+def _get_needed_rank_transform_operator_map_by_layouts(from_tensor_layout, to_tensor_layout, device_list, self_rank,
+                                                       enable_redist_opt=False):
     """
     AllGather op: {op_name, group_ranks + axis}
     """
     stack = []
     index = 0
-    transform_operators = _transform_tensor_by_layout(from_tensor_layout, to_tensor_layout, device_list, self_rank)
+    transform_operators = _transform_tensor_by_layout(from_tensor_layout, to_tensor_layout, device_list, self_rank,
+                                                      enable_redist_opt)
     result_map = {self_rank: transform_operators}
     for operators in transform_operators:
         op_name = operators[0]
@@ -606,7 +610,7 @@ def _get_needed_rank_transform_operator_map_by_layouts(from_tensor_layout, to_te
         for rank in group_info[1]:
             if rank not in result_map:
                 new_transform_operators = _transform_tensor_by_layout(from_tensor_layout, to_tensor_layout,
-                                                                      device_list, rank)
+                                                                      device_list, rank, enable_redist_opt)
                 result_map[rank] = new_transform_operators
                 index = 0
                 for operators in new_transform_operators:
@@ -710,8 +714,6 @@ def _apply_operator(operator_name):
         Returns:
             The data of tensor after apply operator.
         """
-        if str(type(numpy_data)) == "<class 'builtins.PySafeSlice'>":
-            numpy_data = numpy_data[:]
         if not isinstance(numpy_data, np.ndarray):
             raise TypeError("The data should be a numpy.ndarray.")
         _check_operator(reshape_op)
@@ -732,10 +734,7 @@ def _apply_operator(operator_name):
             raise TypeError("The data_list should be a list.")
         new_numpy_data_list = []
         for numpy_data in numpy_data_list:
-            if str(type(numpy_data)) == "<class 'builtins.PySafeSlice'>":
-                new_numpy_data_list.append(numpy_data[:])
-            else:
-                new_numpy_data_list.append(numpy_data)
+            new_numpy_data_list.append(numpy_data)
         numpy_data_list = new_numpy_data_list
         _check_operator(allgather_op)
         concat_group = allgather_op[1][:-1]
@@ -896,3 +895,159 @@ def _chunk_shape(np_tensor, strategy, depth):
         output.extend(
             _chunk_shape(ret_, strategy[len(strategy) - depth + 1:len(strategy)], depth - 1))
     return output
+
+
+def _infer_pp_op_map(from_layout, to_layout, self_rank):
+    """
+    get the ops map for merging pp stages
+    """
+    from_rank_list = from_layout[3]
+    to_rank_list = to_layout[3]
+    from_dev_num_in_stage = len(from_rank_list)
+    current_rank_stage_id = self_rank // from_dev_num_in_stage
+    diff_rank_id = [
+        rank_id for rank_id in to_rank_list if rank_id not in from_rank_list]
+    end_stage = from_dev_num_in_stage * (current_rank_stage_id + 1)
+    start_stage = from_dev_num_in_stage * current_rank_stage_id
+    rank_pos_in_stage = list(range(start_stage, end_stage)).index(self_rank)
+    root_idx = from_rank_list[rank_pos_in_stage]
+    broadcast_rank_list = [root_idx]
+    while rank_pos_in_stage < len(diff_rank_id):
+        broadcast_rank_list.append(diff_rank_id[rank_pos_in_stage])
+        rank_pos_in_stage += from_dev_num_in_stage
+    broadcast_rank_list.sort()
+    broadcast_map = {rank_id: [('Broadcast', root_idx, broadcast_rank_list)] for rank_id in broadcast_rank_list}
+    return broadcast_map
+
+
+def _get_pipeline_operator_map(from_layout, to_layout, self_rank):
+    """
+    If src_pp_stages is greater than dst_pp_stages, the weights of the corresponding cards need to
+    be communicated via broadcast to swap. Need to communicate src rank0's 01 to src rank2,
+    so that rank2 holds param0's data. Similarly, communicate rank1's 02 to rank3
+    rank0 01           01 11
+    rank1 02           02 12
+    pp2 ------->  pp1
+    rank2 11           03 13
+    rank3 12           04 14
+
+    Args:
+        from_layout (tuple): Use tuple to present layout
+          (device_matrix(list), tensor_map(list), global_shape(list), rank_list(list))
+        to_layout (tuple): Use tuple to present layout
+          (device_matrix(list), tensor_map(list), global_shape(list), rank_list(list))
+        self_rank (int): rank_id
+    """
+    if len(from_layout[3]) < len(to_layout[3]):
+        logger.debug(f"from {from_layout} to {to_layout} need to broadcast data across pp stages")
+        comm_tensor_cache_key = (
+            f"{from_layout[0]}, {from_layout[1]}, {from_layout[2]}, {from_layout[3]}"
+            f" -> "
+            f"{to_layout[0]}, {to_layout[1]}, {from_layout[2]}, {to_layout[3]}")
+        global COMM_TENSOR_CELL_CACHE
+        if comm_tensor_cache_key not in COMM_TENSOR_CELL_CACHE:
+            logger.debug(f"comm_tensor_cache_key is {comm_tensor_cache_key}, not match cache")
+            broadcast_map = _infer_pp_op_map(from_layout, to_layout, self_rank)
+            broadcast_op_map_dict = {rank_id: broadcast_map for rank_id in broadcast_map.keys()}
+            COMM_TENSOR_CELL_CACHE[comm_tensor_cache_key] = broadcast_op_map_dict
+        else:
+            comm_tensor_cache_key_rank_list = COMM_TENSOR_CELL_CACHE[comm_tensor_cache_key]
+            if self_rank in comm_tensor_cache_key_rank_list:
+                logger.debug(f"comm_tensor_cache_key is {comm_tensor_cache_key}, match cache")
+                broadcast_map = comm_tensor_cache_key_rank_list[self_rank]
+            else:
+                logger.debug(f"comm_tensor_cache_key is {comm_tensor_cache_key}, but rank {self_rank} not match cache")
+                broadcast_map = _infer_pp_op_map(from_layout, to_layout, self_rank)
+                for rank_id in broadcast_map.keys():
+                    COMM_TENSOR_CELL_CACHE[comm_tensor_cache_key][rank_id] = broadcast_map
+        return broadcast_map
+    logger.debug(f"from {from_layout} to {to_layout} no need to broadcast data across pp stages")
+    return {}
+
+
+def _is_multi_shard(in_tensor_map):
+    """
+    whether the input tensor map is in multi shard
+    """
+    for tensor_map in in_tensor_map:
+        if isinstance(tensor_map, (list, tuple)) and len(tensor_map) > 1:
+            return True
+    return False
+
+
+def _insert_expand_layout_reshape(param_rank_map, from_info_tuple, to_info_tuple,
+                                  insert_from_reshape, insert_to_reshape):
+    """ insert layout expand op reshape """
+    from_dev_matrix = from_info_tuple[0]
+    from_tensor_map = from_info_tuple[1]
+    from_full_tensor_shape = from_info_tuple[2]
+    to_dev_matrix_origin = to_info_tuple[0]
+    to_tensor_map_origin = to_info_tuple[1]
+    origin_tensor_shape = to_info_tuple[2]
+    for param_rank, _ in param_rank_map.items():
+        if insert_from_reshape:
+            from_slice_tensor_shape = ()
+            from_tensor_strategy = _get_tensor_strategy(from_dev_matrix, from_tensor_map)
+            for i, item in enumerate(from_full_tensor_shape):
+                from_slice_tensor_shape += (item // from_tensor_strategy[i],)
+            param_rank_map.get(param_rank).insert(0, ('Reshape', list(from_slice_tensor_shape)))
+        if insert_to_reshape:
+            to_tensor_strategy = _get_tensor_strategy(to_dev_matrix_origin, to_tensor_map_origin)
+            to_slice_tensor_shape = ()
+            for i, item in enumerate(origin_tensor_shape):
+                to_slice_tensor_shape += (item // to_tensor_strategy[i],)
+            param_rank_map.get(param_rank).append(('Reshape', list(to_slice_tensor_shape)))
+
+
+def _infer_reshard_op_map(from_layout, to_layout, self_rank):
+    """infer reshard op map"""
+    from_layout_without_rank_list = from_layout[:-1]
+    to_layout_without_rank_list = to_layout[:-1]
+    if _is_multi_shard(from_layout[1]):
+        # ((2, 1), 1) --> (2, 1, 1) expand tensormap
+        new_layout = _expand_layout(from_layout[0], from_layout[1], from_layout[2])
+        from_layout_without_rank_list = (new_layout[0], new_layout[1], new_layout[2])
+    if _is_multi_shard(to_layout[1]):
+        new_layout = _expand_layout(to_layout[0], to_layout[1], to_layout[2])
+        to_layout_without_rank_list = (new_layout[0], new_layout[1], new_layout[2])
+    operator_map = _get_needed_rank_transform_operator_map_by_layouts(from_layout_without_rank_list,
+                                                                      to_layout_without_rank_list,
+                                                                      from_layout[3], self_rank,
+                                                                      True)
+    new_to_layout_info = to_layout[:-1]
+    _insert_expand_layout_reshape(operator_map, from_layout_without_rank_list, new_to_layout_info,
+                                  _is_multi_shard(from_layout[1]), _is_multi_shard(to_layout[1]))
+    return operator_map
+
+
+def _get_resharding_operator_map(from_layout, to_layout, self_rank):
+    """
+        Args:
+        from_layout (tuple): Use tuple to present layout
+          (device_matrix(list), tensor_map(list), global_shape(list), rank_list(list))
+        to_layout (tuple): Use tuple to present layout
+          (device_matrix(list), tensor_map(list), global_shape(list), rank_list(list))
+        self_rank (int): rank_id
+    """
+    reshard_op_cache_key = (
+        f"{from_layout[0]}, {from_layout[1]}, {from_layout[2]}, {from_layout[3]}"
+        f" -> "
+        f"{to_layout[0]}, {to_layout[1]}, {from_layout[2]}, {to_layout[3]}")
+    global RESHARD_OP_MAP_CACHE
+    if reshard_op_cache_key not in RESHARD_OP_MAP_CACHE:
+        operator_map = _infer_reshard_op_map(from_layout, to_layout, self_rank)
+        op_map_dict = {rank_id: operator_map for rank_id in operator_map}
+        RESHARD_OP_MAP_CACHE[reshard_op_cache_key] = op_map_dict
+        logger.debug(f"reshard_op_cache_key is {reshard_op_cache_key}, not match cache")
+    else:
+        cache_rank_list_dict = RESHARD_OP_MAP_CACHE[reshard_op_cache_key]
+        if self_rank in cache_rank_list_dict:
+            operator_map = cache_rank_list_dict[self_rank]
+            logger.debug(f"reshard_op_cache_key is {reshard_op_cache_key}, match cache")
+        else:
+            logger.debug(f"reshard_op_cache_key is {reshard_op_cache_key}, "
+                         f"but rank {self_rank} is not match cache")
+            operator_map = _infer_reshard_op_map(from_layout, to_layout, self_rank)
+            for rank_id in operator_map:
+                RESHARD_OP_MAP_CACHE[reshard_op_cache_key][rank_id] = operator_map
+    return operator_map

mindspore/parallel/_transformer/moe.py

@@ -111,7 +111,7 @@ def _check_moe_config(moe_config=None, parallel_config=None):
     """
     if not isinstance(moe_config, MoEConfig):
         raise TypeError(f"'moe_config' must be an instance of MoEConfig, but got {type(moe_config).__name__}.")
-    use_moe = (moe_config.expert_num > 1)
+    use_moe = moe_config.expert_num > 1
     if use_moe is False:
         return
     if moe_config.expert_num % parallel_config.expert_parallel != 0:

mindspore/parallel/_transformer/transformer.py

@@ -1365,7 +1365,12 @@ class MultiHeadAttention(Cell):
 class TransformerEncoderLayer(Cell):
     r"""
         Transformer Encoder Layer. This is an implementation of the single layer of the transformer
-        encoder layer, including multihead attention and feedward layer.
+        encoder layer, mainly including Multi-Head Attention, Feed Forward, Add and LayerNorm layer.
+
+        The TransformerEncoderLayer structure is shown in the following figure:
+
+        .. image:: ../images/TransformerEncoderLayer.png
+            :align: center
 
         Args:
             batch_size(int): The batch size of the input tensor when do increnmental prediction. Should be a positive
@@ -1532,7 +1537,7 @@ class TransformerEncoderLayer(Cell):
                     "and parallel_config. model_parallel is {}."
                     .format(ffn_hidden_size, parallel_config.model_parallel))
             _check_moe_config(moe_config, parallel_config)
-            self.use_moe = (moe_config.expert_num > 1)
+            self.use_moe = moe_config.expert_num > 1
             self.use_past = use_past
             self.seq_length = seq_length
             self.hidden_size = hidden_size
@@ -1607,7 +1612,7 @@ class TransformerEncoderLayer(Cell):
                     "and parallel_config. model_parallel is {}."
                     .format(ffn_hidden_size, parallel_config.model_parallel))
             _check_moe_config(moe_config, parallel_config)
-            self.use_moe = (moe_config.expert_num > 1)
+            self.use_moe = moe_config.expert_num > 1
             self.use_past = use_past
             self.seq_length = seq_length
             self.hidden_size = hidden_size
@@ -1902,7 +1907,7 @@ class TransformerDecoderLayer(Cell):
                  parallel_config=default_dpmp_config):
         super(TransformerDecoderLayer, self).__init__()
         _check_moe_config(moe_config, parallel_config)
-        self.use_moe = (moe_config.expert_num > 1)
+        self.use_moe = moe_config.expert_num > 1
         config_to_attention = parallel_config.dpmp if self.use_moe else parallel_config
         if batch_size or use_past:
             Validator.check_positive_int(batch_size)
@@ -2440,7 +2445,7 @@ class TransformerEncoder(Cell):
         super(TransformerEncoder, self).__init__()
         _check_config(parallel_config)
         _check_moe_config(moe_config, parallel_config)
-        self.use_moe = (moe_config.expert_num > 1)
+        self.use_moe = moe_config.expert_num > 1
         config_to_layer = parallel_config.moe_parallel_config if self.use_moe else parallel_config.dp_mp_config
         if _get_parallel_mode() in (ParallelMode.AUTO_PARALLEL,):
             self.add = P.Add()
@@ -2677,7 +2682,7 @@ class TransformerDecoder(Cell):
         super(TransformerDecoder, self).__init__()
         _check_moe_config(moe_config, parallel_config)
         _check_config(parallel_config)
-        self.use_moe = (moe_config.expert_num > 1)
+        self.use_moe = moe_config.expert_num > 1
         config_to_layer = parallel_config.moe_parallel_config if self.use_moe else parallel_config.dp_mp_config
         if _get_parallel_mode() in (ParallelMode.AUTO_PARALLEL,):
             self.add = P.Add()
@@ -2959,7 +2964,7 @@ class Transformer(Cell):
             if not lambda_func:
                 lambda_func = _get_lambda_func(total_layer=encoder_layers + decoder_layers)
             _check_moe_config(moe_config, parallel_config)
-            self.use_moe = (moe_config.expert_num > 1)
+            self.use_moe = moe_config.expert_num > 1
             self.add = P.Add()
             self.aux_loss = Tensor(0.0, mstype.float32)
             if encoder_layers > 0:
@@ -3026,7 +3031,7 @@ class Transformer(Cell):
             if not lambda_func:
                 lambda_func = _get_lambda_func(total_layer=encoder_layers + decoder_layers)
             _check_moe_config(moe_config, parallel_config)
-            self.use_moe = (moe_config.expert_num > 1)
+            self.use_moe = moe_config.expert_num > 1
             self.add = P.Add().shard(((), ()))
             self.aux_loss = Tensor(0.0, mstype.float32)
             if encoder_layers > 0:

mindspore/parallel/auto_parallel.py

@@ -162,6 +162,10 @@ class AutoParallel(Cell):
         super(AutoParallel, self).__init__(auto_prefix=False)
         self.network = network
 
+        if parallel_mode not in ["semi_auto", "sharding_propagation", "recursive_programming"]:
+            raise ValueError("the argument 'parallel_mode' must be one of ['semi_auto', 'sharding_propagation'," \
+                  " 'recursive_programming'], but got the value : {} .".format(parallel_mode))
+
         self._parallel_mode = parallel_mode
 
         self._global_rank = get_rank()
@@ -260,8 +264,10 @@ class AutoParallel(Cell):
         self._save_strategy_file_path = file_path
 
     def disable_strategy_file_only_for_trainable_params(self):
-        """By default, MindSpore only loads and saves trainable parameters. This API enables the loading and saving of
-        non-trainable parameters as well."""
+        """
+        By default, MindSpore only loads and saves trainable parameters. This API enables the loading and saving of
+        non-trainable parameters as well.
+        """
         self._only_trainable_params = False
 
     def save_operator_strategy_file(self, file_path):
@@ -410,7 +416,7 @@ class AutoParallel(Cell):
                 raise ValueError("For 'AutoParallel.dataset_strategy', the argument "
                                  "'config' must be 'full_batch' or 'data_parallel', but got the value : {}."
                                  .format(config))
-            self._full_batch = (config == "full_batch")
+            self._full_batch = config == "full_batch"
             self._dataset_strategy_config = config
             return
         if not isinstance(config, tuple):
@@ -513,9 +519,10 @@ class AutoParallel(Cell):
         if not isinstance(scheduler, str):
             raise TypeError("For 'AutoParallel.pipeline', the argument 'stages' "
                             "must be str type, but got the type : {}.".format(type(scheduler)))
-        if scheduler not in ("1f1b", "gpipe"):
+        if scheduler not in ("1f1b", "gpipe", "seqpipe", "seqvpp", "seqsmartvpp"):
             raise ValueError("For 'AutoParallel.pipeline', the argument "
-                             "'scheduler' must be '1f1b' , 'gpipe' , but got the value : {}."
+                             "'scheduler' must be '1f1b'/'gpipe'/'seqpipe'/'seqvpp'/'seqsmartvpp' ," \
+                             " but got the value : {}."
                              .format(scheduler))
         self._pipeline_stages = stages
         self._pipeline_result_broadcast = output_broadcast

mindspore/parallel/checkpoint_convert.py

@@ -15,10 +15,10 @@
 """Convert distributed checkpoint"""
 from __future__ import absolute_import
 
-from mindspore.parallel.checkpoint_transform import rank_list_for_transform, transform_checkpoint_by_rank, transform_checkpoints
+__all__ = ["rank_list_for_convert", "convert_checkpoint_by_rank", "convert_checkpoints"]
 
-__all__ = ["rank_list_for_convert",
-           "convert_checkpoint_by_rank", "convert_checkpoints"]
+from mindspore.parallel.checkpoint_transform import rank_list_for_transform, transform_checkpoint_by_rank, \
+    transform_checkpoints
 
 
 def rank_list_for_convert(rank_id, src_strategy_file=None, dst_strategy_file=None):

mindspore/parallel/checkpoint_transform.py

@@ -582,6 +582,8 @@ def transform_checkpoints(src_checkpoints_dir, dst_checkpoints_dir, ckpt_prefix,
         The number of multiprocess settings is related to the size of the host, and it is not recommended to set it
         too large, otherwise it may cause freezing.
 
+        This function does not support converting remove_redundancy's checkpoint file.
+
     Args:
         src_checkpoints_dir (str): The source checkpoints directory.
         dst_checkpoints_dir (str): The destination checkpoints directory to save the converted checkpoints.
@@ -1187,7 +1189,7 @@ def load_distributed_checkpoint(network, checkpoint_filenames=None, predict_stra
     param_not_in_ckpt = []
     for _, param in network.parameters_and_names():
         sliced_params = []
-        if param.name not in rank_list.keys():
+        if param.name not in rank_list:
             param_not_in_strategy.append(param.name)
             continue
         if param.name not in param_total_dict:

mindspore/parallel/cluster/process_entity/_api.py

@@ -22,7 +22,8 @@ import socket
 import psutil
 import mindspore.log as logger
 from ._utils import _generate_cmd_args_list, _generate_cmd_args_list_with_core, _generate_url, \
-    _is_local_ip, _convert_addr_to_ip, _send_scale_num, _get_local_ip
+    _is_local_ip, _convert_addr_to_ip, _send_scale_num, _get_local_ip, _generate_auto_bind_core_strategy, \
+    _generate_bind_core_strategy
 
 
 class _Node:
@@ -79,11 +80,12 @@ class _ComputeGraphNode(_Node):
     Worker node for dynamic networking. Inherits from the Node class.
     """
 
-    def __init__(self, worker_num, sched_host, sched_port, timeout, node_id, args_list, output_file,
+    def __init__(self, worker_num, sched_host, sched_port, timeout, node_id, node_rank, args_list, output_file,
                  tail_worker_log, join, is_simulation):
         super().__init__(worker_num, sched_host, sched_port, timeout, args_list, output_file,
                          tail_worker_log, join, is_simulation)
         self.node_id = node_id
+        self.node_rank = node_rank
 
     def run(self):
         """
@@ -95,6 +97,8 @@ class _ComputeGraphNode(_Node):
         super().run()
         if self.node_id is not None:
             os.environ["MS_NODE_ID"] = str(self.node_id)
+        if self.node_rank is not None:
+            os.environ["MS_NODE_RANK"] = str(self.node_rank)
         # If simulation level is set, environment variable 'MS_ROLE' will not be set.
         if not self.is_simulation:
             os.environ["MS_ROLE"] = "MS_WORKER"
@@ -119,6 +123,9 @@ class _ComputeGraphNode(_Node):
         return subprocess.Popen(['/usr/bin/tail', '-f', self.output_file])
 
     def enable_tail_worker_log(self):
+        """
+        Get valid rank ID for tailing the corresponding worker log.
+        """
         tail_worker_log_list = []
         if self.tail_worker_log != "-1":
             tail_worker_log_list.extend([int(num) for num in self.tail_worker_log.split(',')])
@@ -169,7 +176,7 @@ class _ProcessManager:
 
         self.sim_level = args.sim_level
         self.sim_rank_id = args.sim_rank_id
-        self.is_simulation = (self.sim_level != -1)
+        self.is_simulation = self.sim_level != -1
         if self.is_simulation:
             os.environ["MS_SIMULATION_LEVEL"] = str(self.sim_level)
         elif os.getenv("MS_SIMULATION_LEVEL"):
@@ -205,15 +212,24 @@ class _ProcessManager:
             finally:
                 os.umask(origin_mask)
 
+        self.device_to_cpu_map = {}
+        if self.bind_core is True:
+            self.device_to_cpu_map = _generate_auto_bind_core_strategy(self.local_worker_num)
+
         self.proc_rank_map = {}
         self.enable_mindx = False
+        self._check_taskd()
+
+    def _check_taskd(self):
+        """check if enable taskd."""
         tft_env = os.getenv("MS_ENABLE_TFT", "")
-        if ("TTP:1" in tft_env) or ("UCE:1" in tft_env) or ("ARF:1" in tft_env):
+        if any(v in tft_env for v in ('TTP:1', 'UCE:1', 'ARF:1', 'TSP:1', 'RSC:1', 'HCCE:1')):
             try:
                 from taskd.python.framework.agent.ms_mgr.msrun_plugin import MSRunPlugin
                 self.msmgr = MSRunPlugin()
                 self.msmgr.register_callbacks("KILL_WORKER", self.kill_workers)
                 self.msmgr.register_callbacks("START_ALL_WORKER", self.start_all_workers)
+                self.msmgr.register_callbacks("START_WORKER_LIST", self.start_worker_list)
                 self.msmgr.register_callbacks("MONITOR", self.monitor_rank_status)
                 self.enable_mindx = True
                 os.environ["MS_ENABLE_RECOVERY"] = str(1)
@@ -261,6 +277,45 @@ class _ProcessManager:
                               self.is_simulation)
         self.msn_process = msn.run()
 
+    def _start_single_worker(self, local_rank):
+        """
+        Start worker processor
+
+        Args:
+            local_rank: local rank id.
+        """
+        os.environ["DEVICE_ID"] = str(local_rank)
+        node_id, log_name = self._get_node_id_and_log_path(local_rank)
+        if node_id is None:
+            logger.warning(f"Rank ids will be assigned automatically, "
+                           "please use 'grep -rn 'rank id:' command to check each worker log's rank id.")
+        else:
+            # If node_id is generated in '_get_node_id_and_log_path' method, export 'RANK_ID' environment variable.
+            # This is for rank_table method's compatibility consideration.
+            os.environ["RANK_ID"] = str(node_id)
+            print(f"Start worker process with rank id:{node_id}, log file:{log_name}. "
+                  f"Environment variable [RANK_ID={node_id}] is exported.", flush=True)
+        if self.is_simulation and (self.sim_rank_id != -1):
+            # Reset RANK_ID env to sim_rank_id if sim_rank_id is set.
+            os.environ["RANK_ID"] = str(self.sim_rank_id)
+            logger.warning(f"In dryrun case, RANK_ID is assigned to {self.sim_rank_id}.")
+
+        if self.bind_core:
+            affinity_cpu_str = _generate_bind_core_strategy(local_rank, self.device_to_cpu_map, self.bind_core)
+            if affinity_cpu_str is not None:
+                cmd = _generate_cmd_args_list_with_core(self.cmd, self.cmd_args, affinity_cpu_str)
+            else:
+                cmd = _generate_cmd_args_list(self.cmd, self.cmd_args)
+        else:
+            cmd = _generate_cmd_args_list(self.cmd, self.cmd_args)
+        cgn = _ComputeGraphNode(self.worker_num, self.master_addr, self.master_port, self.cluster_time_out,
+                                node_id, self.node_rank, cmd, log_name, self.tail_worker_log, self.join,
+                                self.is_simulation)
+        process, tail_process = cgn.run()
+        self.cgn_processes.append(process)
+        self.tail_cgn_processes.append(tail_process)
+        self.proc_rank_map[local_rank] = process
+
     def start_workers(self):
         """
         Starts the worker nodes.
@@ -275,40 +330,8 @@ class _ProcessManager:
                            "'rank_id' of each process will be assigned after cluster is successfully built.\n"
                            "You can access 'RANK_ID' environment variable after calling "
                            "'mindspore.communication.init()'")
-
         for i in range(self.local_worker_num):
-            os.environ["DEVICE_ID"] = str(i)
-            node_id, log_name = self._get_node_id_and_log_path(i)
-            if node_id is None:
-                logger.warning(f"Rank ids will be assigned automatically, "
-                               "please use 'grep -rn 'rank id:' command to check each worker log's rank id.")
-            else:
-                # If node_id is generated in '_get_node_id_and_log_path' method, export 'RANK_ID' environment variable.
-                # This is for rank_table method's compatibility consideration.
-                os.environ["RANK_ID"] = str(node_id)
-                print(f"Start worker process with rank id:{node_id}, log file:{log_name}. "
-                      f"Environment variable [RANK_ID={node_id}] is exported.", flush=True)
-            if self.is_simulation and (self.sim_rank_id != -1):
-                # Reset RANK_ID env to sim_rank_id if sim_rank_id is set.
-                os.environ["RANK_ID"] = str(self.sim_rank_id)
-                logger.warning(f"In dryrun case, RANK_ID is assigned to {self.sim_rank_id}.")
-
-            if self.bind_core:
-                cpu_num = subprocess.getoutput("cat /proc/cpuinfo|grep processor|wc -l")
-                if not cpu_num.isdigit():
-                    raise RuntimeError(f"Got cpu number from '/proc/cpuinfo' is {cpu_num}, failed to bind core.")
-                avg = int(cpu_num) // self.local_worker_num
-                cpu_start = avg * i
-                cpu_end = cpu_start + avg - 1
-                cmd = _generate_cmd_args_list_with_core(self.cmd, self.cmd_args, cpu_start, cpu_end)
-            else:
-                cmd = _generate_cmd_args_list(self.cmd, self.cmd_args)
-            cgn = _ComputeGraphNode(self.worker_num, self.master_addr, self.master_port, self.cluster_time_out,
-                                    node_id, cmd, log_name, self.tail_worker_log, self.join, self.is_simulation)
-            process, tail_process = cgn.run()
-            self.cgn_processes.append(process)
-            self.tail_cgn_processes.append(tail_process)
-            self.proc_rank_map[i] = process
+            self._start_single_worker(i)
 
     def join_processes(self):
         """
@@ -334,7 +357,7 @@ class _ProcessManager:
                     continue
                 elif ret_code != 0:
                     has_exception = True
-                    logger.error(f"Worker process {p.pid} exit with exception.")
+                    logger.error(f"Worker process {p.pid} exit with exception. Error code: {ret_code}.")
                     break
                 else:
                     success_cgn_processes.add(p)
@@ -420,14 +443,9 @@ class _ProcessManager:
         Args:
             NA.
         """
-        for p in self.cgn_processes:
-            if p.poll() is None:
-                p.kill()
+        self.kill_worker_processes()
+        self.kill_tail_log_processes()
         self.cgn_processes.clear()
-
-        for p in self.tail_cgn_processes:
-            if p is not None:
-                p.kill()
         self.tail_cgn_processes.clear()
 
     def kill_single_worker(self, pid):
@@ -441,7 +459,7 @@ class _ProcessManager:
         for i in range(len(self.cgn_processes)):
             p = self.cgn_processes[i]
             if p.pid == pid and p.poll() is None:
-                p.kill()
+                os.killpg(os.getpgid(p.pid), signal.SIGKILL)
                 del self.cgn_processes[i]
                 tail_p = self.tail_cgn_processes[i]
                 if tail_p is not None:
@@ -499,7 +517,8 @@ class _ProcessManager:
                 p_status = p.poll()
                 if (not psutil.pid_exists(p.pid)) and (p_status != 0):
                     p_status = 300
-                return {"pid": p.pid, "status": p_status, "global_rank": global_rank_id}
+                return {"pid": p.pid, "status": p_status, "global_rank": global_rank_id, "local_rank": rank_id,
+                        "node_id": self.node_rank}
             except KeyError:
                 logger.info(f"Process rank {rank_id} has not been initialized.")
                 return {"pid": None, "status": 200, "global_rank": global_rank_id}
@@ -519,7 +538,24 @@ class _ProcessManager:
         self.start_workers()
         worker_status = self.monitor_rank_status([-1])
         for i in range(self.local_worker_num):
-            if worker_status[i]["status"] != None:  # pylint: disable=singleton-comparison
+            if worker_status[i]["status"] is not None:
+                return 1
+        return 0
+
+    def start_worker_list(self, rank_ids):
+        """
+        Start worker processor by rank list.
+
+        Args:
+            rank_ids: worker process's local rank list, which is also device_id.
+        """
+        if not isinstance(rank_ids, list):
+            raise TypeError(f"The type of 'rank_ids' must be a list, but got:{rank_ids}")
+        for idx in rank_ids:
+            self._start_single_worker(idx)
+        worker_status = self.monitor_rank_status(rank_ids)
+        for i in rank_ids:
+            if worker_status[i]["status"] is not None:
                 return 1
         return 0