mindspore 2.5.0__cp311-cp311-win_amd64.whl → 2.6.0__cp311-cp311-win_amd64.whl

mindspore/parallel/cluster/process_entity/_api.py

@@ -19,15 +19,18 @@ import sys
 import signal
 import subprocess
 import socket
+import psutil
 import mindspore.log as logger
-from ._utils import _generate_cmd_args_list, _generate_cmd_args_list_with_core, _generate_url,\
-                    _is_local_ip, _convert_addr_to_ip, _send_scale_num, _get_local_ip
+from ._utils import _generate_cmd_args_list, _generate_cmd_args_list_with_core, _generate_url, \
+    _is_local_ip, _convert_addr_to_ip, _send_scale_num, _get_local_ip
+
 
 class _Node:
     """
     Base class for dynamic networking nodes.
 
     """
+
     def __init__(self, worker_num, sched_host, sched_port, timeout, args_list, output_file, tail_worker_log,
                  join, is_simulation):
         self.worker_num = worker_num
@@ -40,24 +43,26 @@ class _Node:
         self.join = join
         self.is_simulation = is_simulation
 
-
     def run(self):
         """
         Runs the node by setting environment variables and executing the entrypoint command or script.
 
         """
         os.environ["MS_WORKER_NUM"] = str(self.worker_num)
-        # If simulation level is set, environment variables for dynamic networking will not be set and scheduler will not be started.
+        # If simulation level is set, environment variables for dynamic networking will not be set,
+        # and scheduler will not be started.
         if not self.is_simulation:
             os.environ["MS_SCHED_HOST"] = self.sched_host
             os.environ["MS_SCHED_PORT"] = str(self.sched_port)
             os.environ["MS_TOPO_TIMEOUT"] = str(self.timeout)
 
+
 class _MetaServerNode(_Node):
     """
     Scheduler node for dynamic networking. Inherits from the Node class.
 
     """
+
     def run(self):
         """
         Runs the MetaServerNode by setting environment variables, setting the MS_ROLE variable to
@@ -68,17 +73,18 @@ class _MetaServerNode(_Node):
         with open(self.output_file, "w") as file_handle:
             return subprocess.Popen(self.args_list, stdout=file_handle, stderr=subprocess.STDOUT)
 
+
 class _ComputeGraphNode(_Node):
     """
     Worker node for dynamic networking. Inherits from the Node class.
     """
+
     def __init__(self, worker_num, sched_host, sched_port, timeout, node_id, args_list, output_file,
                  tail_worker_log, join, is_simulation):
         super().__init__(worker_num, sched_host, sched_port, timeout, args_list, output_file,
                          tail_worker_log, join, is_simulation)
         self.node_id = node_id
 
-
     def run(self):
         """
         Runs the ComputeGraphNode by setting environment variables, setting the MS_NODE_ID variable
@@ -127,6 +133,7 @@ class _ProcessManager:
     training
 
     """
+
     def __init__(self, args):
         """
         Initializes a ProcessManager object.
@@ -198,6 +205,21 @@ class _ProcessManager:
             finally:
                 os.umask(origin_mask)
 
+        self.proc_rank_map = {}
+        self.enable_mindx = False
+        tft_env = os.getenv("MS_ENABLE_TFT", "")
+        if ("TTP:1" in tft_env) or ("UCE:1" in tft_env) or ("ARF:1" in tft_env):
+            try:
+                from taskd.python.framework.agent.ms_mgr.msrun_plugin import MSRunPlugin
+                self.msmgr = MSRunPlugin()
+                self.msmgr.register_callbacks("KILL_WORKER", self.kill_workers)
+                self.msmgr.register_callbacks("START_ALL_WORKER", self.start_all_workers)
+                self.msmgr.register_callbacks("MONITOR", self.monitor_rank_status)
+                self.enable_mindx = True
+                os.environ["MS_ENABLE_RECOVERY"] = str(1)
+            except Exception as e:  # pylint: disable=broad-except
+                logger.warning(f"mindx is not installed, using original mindspore recovery strategy.: {str(e)}")
+
     def run(self):
         """
         Runs the process manager.
@@ -218,11 +240,13 @@ class _ProcessManager:
         else:
             if self.is_master and not self.is_simulation:
                 self.start_scheduler()
-        self.start_workers()
-
-        if self.join:
-            logger.warning("Distributed job is spawned. Waiting all processes to exit...")
-            self.join_processes()
+        if self.enable_mindx:
+            self.msmgr.start()
+        else:
+            self.start_workers()
+            if self.join:
+                logger.warning("Distributed job is spawned. Waiting all processes to exit...")
+                self.join_processes()
 
     def start_scheduler(self):
         """
@@ -262,17 +286,17 @@ class _ProcessManager:
                 # If node_id is generated in '_get_node_id_and_log_path' method, export 'RANK_ID' environment variable.
                 # This is for rank_table method's compatibility consideration.
                 os.environ["RANK_ID"] = str(node_id)
-                logger.warning(f"Start worker process with rank id:{node_id}, log file:{log_name}. "
-                               "Environment variable [RANK_ID] is exported.")
+                print(f"Start worker process with rank id:{node_id}, log file:{log_name}. "
+                      f"Environment variable [RANK_ID={node_id}] is exported.", flush=True)
             if self.is_simulation and (self.sim_rank_id != -1):
                 # Reset RANK_ID env to sim_rank_id if sim_rank_id is set.
                 os.environ["RANK_ID"] = str(self.sim_rank_id)
                 logger.warning(f"In dryrun case, RANK_ID is assigned to {self.sim_rank_id}.")
 
-            cpu_num = subprocess.getoutput("cat /proc/cpuinfo|grep processor|wc -l")
-            if not cpu_num.isdigit():
-                raise RuntimeError("Fail to get cpu number from /proc/cpuinfo.")
             if self.bind_core:
+                cpu_num = subprocess.getoutput("cat /proc/cpuinfo|grep processor|wc -l")
+                if not cpu_num.isdigit():
+                    raise RuntimeError(f"Got cpu number from '/proc/cpuinfo' is {cpu_num}, failed to bind core.")
                 avg = int(cpu_num) // self.local_worker_num
                 cpu_start = avg * i
                 cpu_end = cpu_start + avg - 1
@@ -284,7 +308,7 @@ class _ProcessManager:
             process, tail_process = cgn.run()
             self.cgn_processes.append(process)
             self.tail_cgn_processes.append(tail_process)
-
+            self.proc_rank_map[i] = process
 
     def join_processes(self):
         """
@@ -292,6 +316,7 @@ class _ProcessManager:
         If there's any process does not exit normally, logs will be analyzed
         so that understandable root cause of exception could be returned.
         """
+
         def signal_handler(sig, frame):
             logger.warning("msrun process received SIGNIN (Ctrl+C), terminating all workers.")
             self.kill_all_processes()
@@ -331,7 +356,7 @@ class _ProcessManager:
                 logger.error(f"Scheduler process {self.msn_process.pid} exit with exception.")
 
         if has_exception:
-            logger.warning("Analyzing exception log...")
+            logger.info("Analyzing exception log...")
             self._analyze_log()
             raise RuntimeError("Distributed job exited with exception. Please check logs in "
                                f"directory: {self.log_dir}.")
@@ -388,6 +413,115 @@ class _ProcessManager:
             self.start_scheduler()
         self.start_workers()
 
+    def kill_all_workers(self):
+        """
+        Kill all running worker processes.
+
+        Args:
+            NA.
+        """
+        for p in self.cgn_processes:
+            if p.poll() is None:
+                p.kill()
+        self.cgn_processes.clear()
+
+        for p in self.tail_cgn_processes:
+            if p is not None:
+                p.kill()
+        self.tail_cgn_processes.clear()
+
+    def kill_single_worker(self, pid):
+        """
+        Kill one worker process with specified pid.
+
+        Args:
+            pid: Worker process' pid.
+        """
+        kill_status = False
+        for i in range(len(self.cgn_processes)):
+            p = self.cgn_processes[i]
+            if p.pid == pid and p.poll() is None:
+                p.kill()
+                del self.cgn_processes[i]
+                tail_p = self.tail_cgn_processes[i]
+                if tail_p is not None:
+                    tail_p.kill()
+                del self.tail_cgn_processes[i]
+                kill_status = True
+                break
+        if not kill_status:
+            logger.warning(f"There's no active worker with pid: {pid}")
+
+    def kill_workers(self, pids):
+        """
+        Kill worker process according to pids. Worker process with pid within pids list will be killed.
+
+        Args:
+            pids(list): a list of worker process pid. When local_ranks pids -1, kill all worker process.
+        """
+        if -1 in pids:
+            self.kill_all_workers()
+        else:
+            for pid in pids:
+                self.kill_single_worker(pid)
+        return 0
+
+    def monitor_rank_status(self, local_ranks):
+        """
+        Monitor the status of workers whose rank is within local_ranks list.
+
+        Args:
+            local_ranks(list): a list of local worker ranks. When local_ranks contains -1,
+                monitor all workers' status.
+        """
+        rank_status = {}
+        if -1 in local_ranks:
+            local_ranks = list(range(self.local_worker_num))
+        for i in local_ranks:
+            single_status = self.monitor_single_rank(i)
+            if single_status:
+                rank_status[i] = single_status
+        return rank_status
+
+    def monitor_single_rank(self, rank_id):
+        """
+        Monitor the status of a single worker with rank_id
+
+        Args:
+            rank_id: worker process's local rank, which is also device_id.
+        """
+        if 0 <= rank_id < self.local_worker_num:
+            global_rank_id = rank_id
+            if self.node_rank >= 0:
+                global_rank_id = self.node_rank * self.local_worker_num + rank_id
+            try:
+                p = self.proc_rank_map[rank_id]
+                p_status = p.poll()
+                if (not psutil.pid_exists(p.pid)) and (p_status != 0):
+                    p_status = 300
+                return {"pid": p.pid, "status": p_status, "global_rank": global_rank_id}
+            except KeyError:
+                logger.info(f"Process rank {rank_id} has not been initialized.")
+                return {"pid": None, "status": 200, "global_rank": global_rank_id}
+        else:
+            logger.warning(f"Invalid rank id!")
+        return {}
+
+    def start_all_workers(self):
+        """
+        Start all worker processes after killing all workers.
+
+        Args:
+            NA.
+        """
+        if self.cgn_processes:
+            self.kill_all_workers()
+        self.start_workers()
+        worker_status = self.monitor_rank_status([-1])
+        for i in range(self.local_worker_num):
+            if worker_status[i]["status"] != None:  # pylint: disable=singleton-comparison
+                return 1
+        return 0
 
     def _get_node_id_and_log_path(self, index):
         """
@@ -410,7 +544,6 @@ class _ProcessManager:
             log_name = os.path.join(self.log_dir, formatted_log_name + "_" + str(index) + ".log")
         return node_id, log_name
 
-
     def _analyze_log(self):
         """
         Analyze exception logs.
@@ -432,7 +565,6 @@ class _ProcessManager:
 
         os.system(f"grep -rn -E 'ERROR|CRITICAL|Traceback|Error' -C 5 {self.log_dir}")
 
-
     def format_worker_log_name(self):
         """
         Format worker log files' name.

mindspore/parallel/cluster/run.py

@@ -85,7 +85,7 @@ def get_args():
         "--sim_level",
         default=-1,
         type=int,
-        choices=[0, 1],
+        choices=[0, 1, 2, 3],
         help="specifies simulation level. When this argument is set, msrun only spawns one process "
              "but export RANK_SIZE with value worker_num and RANK_ID with value sim_rank_id."
     )

mindspore/parallel/function/__init__.py

@@ -0,0 +1,24 @@
+# Copyright 2025 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+"""
+Parallel function operator
+"""
+
+from mindspore.parallel.function.reshard_func import reshard
+
+__all__ = []
+__all__.extend(reshard_func.__all__)
+__all__.sort()

mindspore/parallel/function/reshard_func.py

@@ -0,0 +1,258 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Defines parameter operators with functional form."""
+from mindspore import context, ops
+from mindspore import log as logger
+from mindspore.ops import operations as P
+from mindspore.ops._primitive_cache import _get_cache_prim
+from mindspore.common.tensor import Tensor
+from mindspore.communication.management import get_group_size, get_rank
+from mindspore.parallel.shard import Layout, _DistributedTensorInfo
+from mindspore.parallel._auto_parallel_context import _get_all_auto_parallel_context, _recover_auto_parallel_context
+from mindspore.ops.primitive import constexpr
+
+
+REDIST_CELL_CACHE = {}
+COMM_TENSOR_CELL_CACHE = {}
+
+
+@constexpr
+def group_size():
+    """ Return the device number in the Cell's construct method. """
+    return get_group_size()
+
+
+# pylint: disable=W0212
+def reshard(tensor, layout):
+    r"""
+    Converting a tensor from one distributed arrangement to another distributed arrangement.
+    The given layout must be type mindspore.parallel.Layout,
+    can check :class:`mindspore.parallel.Layout` for reference.
+
+    Note:
+        - In the Graph mode, this function can set the sharding propagation strategy of a tensor.
+          For those tensor do not manually be set, their strategies are decided by the sharding
+          strategy propagation algorithm automatically.
+        - In PyNative mode, you can use this method to arrange tensors in a cell (that is, cells
+          that use Cell.shard/F.shard in PyNative mode) that is executed in parallel in graph mode.
+
+    Args:
+        tensor (Tensor): The tensor to be set the sharding strategy.
+        layout (Layout): The layout to shard the tensor precisely, including the device
+                         arrangement (device_matrix) and the alias for the device matrix
+                         (alias_name).
+
+    Returns:
+        Tensor. The mathematically equivalent of the input tensor.
+
+    Raises:
+        TypeError: If the type of input param `tensor` is not mindspore.Tensor.
+        TypeError: If the type of input param `layout` is not mindspore.parallel.Layout.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        .. note::
+            Before running the following examples, you need to configure the communication environment variables.
+
+            For Ascend/GPU/CPU devices, it is recommended to use the msrun startup method
+            without any third-party or configuration file dependencies.
+            Please see the `msrun start-up
+            <https://www.mindspore.cn/tutorials/en/master/parallel/msrun_launcher.html>`_
+            for more details.
+
+            This example should be run with 8 devices.
+
+        >>> import numpy as np
+        >>> import mindspore as ms
+        >>> from mindspore import ops, nn, Tensor, context, Layout
+        >>> from mindspore.parallel.function import reshard
+        >>> from mindspore.nn.utils import no_init_parameters
+        >>> from mindspore.parallel.auto_parallel import AutoParallel
+        >>> from mindspore.communication import init
+        >>> context.set_context(mode=ms.GRAPH_MODE)
+        >>> init()
+        >>> class Network(nn.Cell):
+        ...     def __init__(self):
+        ...         super().__init__()
+        ...         self.matmul = ops.MatMul()
+        ...         self.relu = ops.ReLU()
+        ...     def construct(self, x, layout):
+        ...         x = self.relu(x)
+        ...         x_reshard = reshard(x, layout)
+        ...         y = Tensor(np.ones(shape=(128, 128)), dtype=ms.float32)
+        ...         x = self.matmul(x_reshard, y)
+        ...         return x
+        >>> layout = Layout((4, 2), ("dp", "mp"))
+        >>> input_layout = layout("dp", "mp")
+        >>> with no_init_parameters():
+        ...     net = Network()
+        >>> parallel_net = AutoParallel(net, parallel_mode='sharding_propagation')
+        >>> tensor = Tensor(np.ones(shape=(128, 128)), dtype=ms.float32)
+        >>> out = parallel_net(tensor, input_layout)
+    """
+    if group_size() == 1:
+        return tensor
+    if not isinstance(tensor, Tensor):
+        raise TypeError(f"Reshard takes in Tensor type as the first input param, but got: {type(tensor)}.")
+    if not isinstance(layout, Layout):
+        raise TypeError(f"Reshard only support type mindspore.parallel.Layout, but got: {type(layout)}.")
+
+    def layout_to_tuple(layout):
+        layout_dict = layout.to_dict()
+        tensor_map = layout_dict["tensor_map"]
+        device_matrix_rev = layout_dict["device_matrix"][::-1]
+        axis_stgy = ()
+        for ind in tensor_map:
+            if ind == -1:
+                axis_stgy += (1,)
+            else:
+                axis_stgy += (device_matrix_rev[ind],)
+        return axis_stgy
+
+    in_strategy = layout_to_tuple(layout)
+    _reshard = _get_cache_prim(P.Reshard)(in_layout=(layout,), out_layout=(layout,), in_strategy=(in_strategy,))
+    return _reshard(tensor)
+
+
+def _redistribute(tensor, dst_dtensor_info):
+    """
+    Redistribute the tensor from the source sharding strategy to the destination sharding strategy.
+
+    Args:
+        tensor (Tensor): The source tensor.
+        dst_dtensor_info (_DistributedTensorInfo): The destination sharding strategy.
+
+    Returns:
+        Tensor, value is same as the source tensor, but the sharding strategy is the destination sharding strategy.
+
+    Supported Platforms:
+        ``Ascend``
+
+    Examples:
+        .. note::
+            Before running the following examples, you need to configure the communication environment variables.
+
+            For Ascend/GPU/CPU devices, it is recommended to use the msrun startup method
+            without any third-party or configuration file dependencies.
+            Please see the `msrun start up
+            <https://www.mindspore.cn/tutorials/en/master/parallel/msrun_launcher.html>`_
+            for more details.
+
+            This example should be run with 2 devices.
+
+        >>> import numpy as np
+        >>> from mindspore.communication import init
+        >>> from mindspore import Tensor, Layout, _DistributedTensorInfo
+        >>>
+        >>> init()
+        >>> layout = Layout((2, 1), ("dp", "mp"))
+        >>> src_layout = layout("dp", "mp")
+        >>> distributed_info = _DistributedTensorInfo(src_layout)
+        >>> x = Tensor(np.ones([2, 2]).astype(np.float32))
+        >>> out = x.redistribute(distributed_info)
+        >>> print(out)
+        [[1. 1.]]
+    """
+    from mindspore.parallel._cell_wrapper import RedistributionCell, _insert_virtual_pp_dim
+    if not isinstance(dst_dtensor_info, _DistributedTensorInfo):
+        raise TypeError(
+            "dst_dtensor_info should be _DistributedTensorInfo type, but got {}".format(type(dst_dtensor_info)))
+    run_mode = context.get_context("mode")
+    context.set_context(mode=context.GRAPH_MODE)
+    og_auto_parallel_context, pp_config = _get_all_auto_parallel_context()
+    context.reset_auto_parallel_context()
+    tensor_data = tensor
+    all_reduce_data = False
+    # If src_pp_stages is less than or equal to dst_pp_stages, the parameters of each pp stage of src can be
+    # directly swapped to the corresponding card of dst
+    # rank0 01 11           01
+    # rank1 02 12           02
+    #  pp1   ------>  pp2
+    # rank2 03 13           11
+    # rank3 04 14           12
+    # if dtensor info is None, return the all 1 strategy as from dtensor info
+    if tensor._dtensor_info is None:
+        all_dev_num = get_group_size()
+        dev_mat = Layout((all_dev_num,), ("replica",))
+        tensor_map = ["None"] * len(tensor.shape)
+        layout = dev_mat(*tensor_map)
+        tensor._dtensor_info = _DistributedTensorInfo(layout)
+    src_layout_info = tensor._dtensor_info.layout.to_dict()
+    dst_layout_info = dst_dtensor_info.layout.to_dict()
+    if len(tensor._dtensor_info.layout.to_dict()["rank_list"]) < len(dst_dtensor_info.layout.to_dict()["rank_list"]):
+        # If src_pp_stages is greater than dst_pp_stages, the weights of the corresponding cards need to
+        # be communicated via AllReduce to swap. Need to communicate src rank0's 01 to src rank2,
+        # so that rank2 holds param0's data. Similarly, communicate rank1's 02 to rank3
+        # rank0 01           01 11
+        # rank1 02           02 12
+        # pp2 ------->  pp1
+        # rank2 11           03 13
+        # rank3 12           04 14
+        from mindspore.parallel._cell_wrapper import CommTensorDataForPP
+        if get_rank() in dst_dtensor_info.layout.to_dict()["rank_list"]:
+            comm_tensor_cache_key = (
+                f"{src_layout_info['device_matrix']}, {src_layout_info['tensor_map']}, {src_layout_info['rank_list']}"
+                f" -> "
+                f"{dst_layout_info['device_matrix']}, {dst_layout_info['tensor_map']}, {dst_layout_info['rank_list']}")
+            global COMM_TENSOR_CELL_CACHE
+            if comm_tensor_cache_key not in COMM_TENSOR_CELL_CACHE:
+                comm_tensor_data_func = CommTensorDataForPP(tensor._dtensor_info, dst_dtensor_info)
+                COMM_TENSOR_CELL_CACHE[comm_tensor_cache_key] = comm_tensor_data_func
+                logger.debug(f"comm_tensor_cache_key is {comm_tensor_cache_key}, not match cache")
+            else:
+                comm_tensor_data_func = COMM_TENSOR_CELL_CACHE[comm_tensor_cache_key]
+                logger.debug(f"comm_tensor_cache_key is {comm_tensor_cache_key}, match cache")
+            if not comm_tensor_data_func._current_rank_has_data:
+                new_tensor_shape = tuple([tensor_data.shape[i] // tensor._dtensor_info.sharding_strategy[i]
+                                          for i in range(len(tensor.shape))])
+                tensor_data = ops.zeros(new_tensor_shape, tensor.dtype)
+                _ = comm_tensor_data_func.comm_data(tensor_data)
+            else:
+                _ = comm_tensor_data_func.comm_data(tensor_data)
+            all_reduce_data = True
+    if src_layout_info['device_matrix'] == dst_layout_info['device_matrix'] and src_layout_info['tensor_map'] == \
+            dst_layout_info['tensor_map']:
+        return tensor_data
+    dataset_strategy = (_insert_virtual_pp_dim(tensor._dtensor_info.layout),)
+    if get_rank() not in tensor._dtensor_info.layout.to_dict()["rank_list"] and not all_reduce_data:
+        dataset_strategy = "full_batch"
+    context.set_auto_parallel_context(dataset_strategy=dataset_strategy,
+                                      parallel_mode="semi_auto_parallel", device_num=get_group_size())
+    global REDIST_CELL_CACHE
+    redist_cache_key = (f"{src_layout_info['device_matrix']}, {src_layout_info['tensor_map']} -> "
+                        f"{dst_layout_info['device_matrix']}, {dst_layout_info['tensor_map']}")
+    if redist_cache_key in REDIST_CELL_CACHE.keys():
+        logger.debug(f"redist_cache_key is {redist_cache_key}, match cache")
+        redist_func = REDIST_CELL_CACHE[redist_cache_key]
+    else:
+        logger.debug(f"redist_cache_key is {redist_cache_key}, not match cache")
+        redist_func = RedistributionCell(tensor._dtensor_info.layout, dst_dtensor_info.layout)
+        REDIST_CELL_CACHE[redist_cache_key] = redist_func
+    redist_func.set_train(True)
+    redist_tensor_data = redist_func(tensor_data)
+    context.reset_auto_parallel_context()
+    _recover_auto_parallel_context(og_auto_parallel_context, pp_config)
+    context.set_context(mode=run_mode)
+    redist_tensor_data._dtensor_info = dst_dtensor_info
+    return redist_tensor_data
+
+
+__all__ = [
+    'reshard'
+]
+
+__all__.sort()

mindspore/parallel/nn/__init__.py

@@ -0,0 +1,25 @@
+# Copyright 2025 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""
+Interfaces for parallel-related functionality
+"""
+from __future__ import absolute_import
+
+from mindspore.parallel.nn.parallel_grad_reducer import PipelineGradReducer
+from mindspore.parallel.nn.parallel_cell_wrapper import PipelineCell, Pipeline, MicroBatchInterleaved, GradAccumulation
+
+__all__ = []
+__all__.extend(parallel_grad_reducer.__all__)
+__all__.extend(parallel_cell_wrapper.__all__)