mindspore 2.4.10__cp311-cp311-win_amd64.whl → 2.6.0rc1__cp311-cp311-win_amd64.whl

mindspore/parallel/cluster/process_entity/_api.py

@@ -16,23 +16,32 @@
 import os
 import re
 import sys
+import signal
 import subprocess
+import socket
+import psutil
 import mindspore.log as logger
-from ._utils import _generate_cmd_args_list, _generate_cmd_args_list_with_core, _generate_url,\
-                    _is_local_ip, _send_scale_num
+from ._utils import _generate_cmd_args_list, _generate_cmd_args_list_with_core, _generate_url, \
+    _is_local_ip, _convert_addr_to_ip, _send_scale_num, _get_local_ip
+
 
 class _Node:
     """
     Base class for dynamic networking nodes.
 
     """
-    def __init__(self, worker_num, sched_host, sched_port, timeout, args_list, output_file):
+
+    def __init__(self, worker_num, sched_host, sched_port, timeout, args_list, output_file, tail_worker_log,
+                 join, is_simulation):
         self.worker_num = worker_num
         self.sched_host = sched_host
         self.sched_port = sched_port
         self.args_list = args_list
         self.output_file = output_file
         self.timeout = timeout
+        self.tail_worker_log = tail_worker_log
+        self.join = join
+        self.is_simulation = is_simulation
 
     def run(self):
         """
@@ -40,15 +49,20 @@ class _Node:
 
         """
         os.environ["MS_WORKER_NUM"] = str(self.worker_num)
-        os.environ["MS_SCHED_HOST"] = self.sched_host
-        os.environ["MS_SCHED_PORT"] = str(self.sched_port)
-        os.environ["MS_TOPO_TIMEOUT"] = str(self.timeout)
+        # If simulation level is set, environment variables for dynamic networking will not be set,
+        # and scheduler will not be started.
+        if not self.is_simulation:
+            os.environ["MS_SCHED_HOST"] = self.sched_host
+            os.environ["MS_SCHED_PORT"] = str(self.sched_port)
+            os.environ["MS_TOPO_TIMEOUT"] = str(self.timeout)
+
 
 class _MetaServerNode(_Node):
     """
     Scheduler node for dynamic networking. Inherits from the Node class.
 
     """
+
     def run(self):
         """
         Runs the MetaServerNode by setting environment variables, setting the MS_ROLE variable to
@@ -59,14 +73,17 @@ class _MetaServerNode(_Node):
         with open(self.output_file, "w") as file_handle:
             return subprocess.Popen(self.args_list, stdout=file_handle, stderr=subprocess.STDOUT)
 
+
 class _ComputeGraphNode(_Node):
     """
     Worker node for dynamic networking. Inherits from the Node class.
     """
-    def __init__(self, worker_num, sched_host, sched_port, timeout, node_id, args_list, output_file):
-        super().__init__(worker_num, sched_host, sched_port, timeout, args_list, output_file)
-        self.node_id = node_id
 
+    def __init__(self, worker_num, sched_host, sched_port, timeout, node_id, args_list, output_file,
+                 tail_worker_log, join, is_simulation):
+        super().__init__(worker_num, sched_host, sched_port, timeout, args_list, output_file,
+                         tail_worker_log, join, is_simulation)
+        self.node_id = node_id
 
     def run(self):
         """
@@ -78,9 +95,36 @@ class _ComputeGraphNode(_Node):
         super().run()
         if self.node_id is not None:
             os.environ["MS_NODE_ID"] = str(self.node_id)
-        os.environ["MS_ROLE"] = "MS_WORKER"
+        # If simulation level is set, environment variable 'MS_ROLE' will not be set.
+        if not self.is_simulation:
+            os.environ["MS_ROLE"] = "MS_WORKER"
+        tail_worker_process = None
+        is_tail_worker_log = self.enable_tail_worker_log()
+        if self.join and not is_tail_worker_log:
+            logger.warning(f"The '--tail_worker_log' is:{self.tail_worker_log}, "
+                           f"which doesn't contain this worker {self.node_id}."
+                           f" So this worker {self.node_id}'s log will not be output to console. Reset "
+                           "'--tail_worker_log', if you want to output this worker's log to console.")
         with open(self.output_file, "w") as file_handle:
-            return subprocess.Popen(self.args_list, stdout=file_handle, stderr=subprocess.STDOUT)
+            worker_process = subprocess.Popen(self.args_list, preexec_fn=os.setsid, stdout=file_handle,
+                                              stderr=subprocess.STDOUT)
+            if self.join and is_tail_worker_log:
+                tail_worker_process = self.output_to_console()
+            return worker_process, tail_worker_process
+
+    def output_to_console(self):
+        """
+        Output worker log file to console.
+        """
+        return subprocess.Popen(['/usr/bin/tail', '-f', self.output_file])
+
+    def enable_tail_worker_log(self):
+        tail_worker_log_list = []
+        if self.tail_worker_log != "-1":
+            tail_worker_log_list.extend([int(num) for num in self.tail_worker_log.split(',')])
+        if self.tail_worker_log != "-1" and self.node_id not in tail_worker_log_list:
+            return False
+        return True
 
 
 class _ProcessManager:
@@ -89,6 +133,7 @@ class _ProcessManager:
     training
 
     """
+
     def __init__(self, args):
         """
         Initializes a ProcessManager object.
@@ -99,13 +144,14 @@ class _ProcessManager:
         """
         self.msn_process = None
         self.cgn_processes = []
+        self.tail_cgn_processes = []
 
-        """`is_master` flags whether the current node is the master node."""
-        self.is_master = _is_local_ip(args.master_addr)
-
-        self.master_addr = args.master_addr
+        self.master_addr = _convert_addr_to_ip(args.master_addr)
         self.master_port = args.master_port
 
+        """`is_master` flags whether the current node is the master node."""
+        self.is_master = _is_local_ip(self.master_addr)
+
         self.worker_num = args.worker_num
         if self.worker_num <= 0:
             raise ValueError(f"worker_num must be greater than 0, but got {self.worker_num}.")
@@ -115,6 +161,8 @@ class _ProcessManager:
 
         self.log_dir = args.log_dir
         self.join = args.join
+        self.worker_log_name = args.worker_log_name
+        self.tail_worker_log = args.tail_worker_log
         self.cluster_time_out = args.cluster_time_out
         self.bind_core = args.bind_core
         self.rank_table_file = args.rank_table_file
@@ -123,19 +171,21 @@ class _ProcessManager:
         self.sim_rank_id = args.sim_rank_id
         self.is_simulation = (self.sim_level != -1)
         if self.is_simulation:
-            # If simulation level is set, reset the worker_num and local_worker_num to 1
-            # so that host cluster could be initialized.
-            self.worker_num = 1
-            self.local_worker_num = 1
             os.environ["MS_SIMULATION_LEVEL"] = str(self.sim_level)
         elif os.getenv("MS_SIMULATION_LEVEL"):
-            # If simulation level env is set, load RANK_ID and RANK_SIZE envs.
-            self.worker_num = 1
-            self.local_worker_num = 1
             self.is_simulation = True
-            self.sim_rank_id = os.getenv("RANK_ID", "0")
+            self.sim_rank_id = int(os.getenv("RANK_ID", "-1"))
             if os.getenv("RANK_SIZE"):
                 self.exported_rank_size = os.getenv("RANK_SIZE")
+        # If sim_rank_id is set, single worker can be started.
+        if self.is_simulation and (self.sim_rank_id != -1):
+            logger.info(f"Simulation rank id is set to {self.sim_rank_id}, will dryrun a single process.")
+            self.local_worker_num = 1
+        if self.is_simulation and self.local_worker_num > 128:
+            self.local_worker_num = 1
+            self.sim_rank_id = 0
+            logger.warning(f"In dryrun case, local worker num is set to larger than 128. "
+                           "To avoid a system clash, local worker num is set to 1.")
 
         self.cmd = args.task_script
         self.cmd_args = args.task_script_args
@@ -155,6 +205,21 @@ class _ProcessManager:
             finally:
                 os.umask(origin_mask)
 
+        self.proc_rank_map = {}
+        self.enable_mindx = False
+        tft_env = os.getenv("MS_ENABLE_TFT", "")
+        if ("TTP:1" in tft_env) or ("UCE:1" in tft_env) or ("ARF:1" in tft_env):
+            try:
+                from taskd.python.framework.agent.ms_mgr.msrun_plugin import MSRunPlugin
+                self.msmgr = MSRunPlugin()
+                self.msmgr.register_callbacks("KILL_WORKER", self.kill_workers)
+                self.msmgr.register_callbacks("START_ALL_WORKER", self.start_all_workers)
+                self.msmgr.register_callbacks("MONITOR", self.monitor_rank_status)
+                self.enable_mindx = True
+                os.environ["MS_ENABLE_RECOVERY"] = str(1)
+            except Exception as e:  # pylint: disable=broad-except
+                logger.warning(f"mindx is not installed, using original mindspore recovery strategy.: {str(e)}")
+
     def run(self):
         """
         Runs the process manager.
@@ -173,13 +238,15 @@ class _ProcessManager:
             else:
                 sys.exit()
         else:
-            if self.is_master:
+            if self.is_master and not self.is_simulation:
                 self.start_scheduler()
-        self.start_workers()
-
-        if self.join:
-            logger.warning("Distributed job is spawned. Waiting all processes to exit...")
-            self.join_processes()
+        if self.enable_mindx:
+            self.msmgr.start()
+        else:
+            self.start_workers()
+            if self.join:
+                logger.warning("Distributed job is spawned. Waiting all processes to exit...")
+                self.join_processes()
 
     def start_scheduler(self):
         """
@@ -190,7 +257,8 @@ class _ProcessManager:
         os.environ['RANK_ID'] = str(0)
         msn = _MetaServerNode(self.worker_num, self.master_addr, self.master_port, self.cluster_time_out,
                               _generate_cmd_args_list(self.cmd, self.cmd_args),
-                              os.path.join(self.log_dir, "scheduler.log"))
+                              os.path.join(self.log_dir, "scheduler.log"), self.tail_worker_log, self.join,
+                              self.is_simulation)
         self.msn_process = msn.run()
 
     def start_workers(self):
@@ -208,9 +276,6 @@ class _ProcessManager:
                            "You can access 'RANK_ID' environment variable after calling "
                            "'mindspore.communication.init()'")
 
-        if self.is_simulation and self.worker_num != 1:
-            raise ValueError(f"Simulation level is set, worker_num must be 1, but got {self.worker_num}.")
-
         for i in range(self.local_worker_num):
             os.environ["DEVICE_ID"] = str(i)
             node_id, log_name = self._get_node_id_and_log_path(i)
@@ -221,16 +286,17 @@ class _ProcessManager:
                 # If node_id is generated in '_get_node_id_and_log_path' method, export 'RANK_ID' environment variable.
                 # This is for rank_table method's compatibility consideration.
                 os.environ["RANK_ID"] = str(node_id)
-                logger.warning(f"Start worker process with rank id:{node_id}, log file:{log_name}. "
-                               "Environment variable [RANK_ID] is exported.")
-            if self.is_simulation:
-                # Reset RANK_ID env to sim_rank_id.
+                print(f"Start worker process with rank id:{node_id}, log file:{log_name}. "
+                      f"Environment variable [RANK_ID={node_id}] is exported.", flush=True)
+            if self.is_simulation and (self.sim_rank_id != -1):
+                # Reset RANK_ID env to sim_rank_id if sim_rank_id is set.
                 os.environ["RANK_ID"] = str(self.sim_rank_id)
+                logger.warning(f"In dryrun case, RANK_ID is assigned to {self.sim_rank_id}.")
 
-            cpu_num = subprocess.getoutput("cat /proc/cpuinfo|grep processor|wc -l")
-            if not cpu_num.isdigit():
-                raise RuntimeError("Fail to get cpu number from /proc/cpuinfo.")
             if self.bind_core:
+                cpu_num = subprocess.getoutput("cat /proc/cpuinfo|grep processor|wc -l")
+                if not cpu_num.isdigit():
+                    raise RuntimeError(f"Got cpu number from '/proc/cpuinfo' is {cpu_num}, failed to bind core.")
                 avg = int(cpu_num) // self.local_worker_num
                 cpu_start = avg * i
                 cpu_end = cpu_start + avg - 1
@@ -238,9 +304,11 @@ class _ProcessManager:
             else:
                 cmd = _generate_cmd_args_list(self.cmd, self.cmd_args)
             cgn = _ComputeGraphNode(self.worker_num, self.master_addr, self.master_port, self.cluster_time_out,
-                                    node_id, cmd, log_name)
-            process = cgn.run()
+                                    node_id, cmd, log_name, self.tail_worker_log, self.join, self.is_simulation)
+            process, tail_process = cgn.run()
             self.cgn_processes.append(process)
+            self.tail_cgn_processes.append(tail_process)
+            self.proc_rank_map[i] = process
 
     def join_processes(self):
         """
@@ -248,8 +316,15 @@ class _ProcessManager:
         If there's any process does not exit normally, logs will be analyzed
         so that understandable root cause of exception could be returned.
         """
+
+        def signal_handler(sig, frame):
+            logger.warning("msrun process received SIGNIN (Ctrl+C), terminating all workers.")
+            self.kill_all_processes()
+            sys.exit(0)
+
         has_exception = False
         success_cgn_processes = set()
+        signal.signal(signal.SIGINT, signal_handler)
         while True:
             # Traversal all workers and kill immediately if any exception happens.
             for p in self.cgn_processes:
@@ -266,15 +341,14 @@ class _ProcessManager:
 
             if has_exception:
                 logger.warning("There's worker exits with exception, kill all other workers.")
-                for p in self.cgn_processes:
-                    if p.poll() is None:
-                        p.kill()
+                self.kill_worker_processes()
+                self.kill_tail_log_processes()
                 break
             elif len(success_cgn_processes) == len(self.cgn_processes):
                 logger.info("All workers successfully exit!")
+                self.kill_tail_log_processes()
                 break
 
-
         if self.msn_process:
             self.msn_process.wait()
             if self.msn_process.returncode != 0:
@@ -282,11 +356,40 @@ class _ProcessManager:
                 logger.error(f"Scheduler process {self.msn_process.pid} exit with exception.")
 
         if has_exception:
-            logger.warning("Analyzing exception log...")
+            logger.info("Analyzing exception log...")
             self._analyze_log()
             raise RuntimeError("Distributed job exited with exception. Please check logs in "
                                f"directory: {self.log_dir}.")
 
+    def kill_tail_log_processes(self):
+        """
+        Kills all tail worker log processes.
+
+        """
+        for p_tail in self.tail_cgn_processes:
+            if p_tail is not None:
+                logger.debug("Tail worker log process:{p_tail.pid} has been killed!")
+                p_tail.kill()
+
+    def kill_worker_processes(self):
+        """
+        Kills all worker processes.
+
+        """
+        for p in self.cgn_processes:
+            if p.poll() is None:
+                os.killpg(os.getpgid(p.pid), signal.SIGKILL)
+
+    def kill_all_processes(self):
+        """
+        Kills all running processes, including scheduler, worker and tail log.
+
+        """
+        self.kill_worker_processes()
+        self.kill_tail_log_processes()
+        if self.msn_process.poll() is None:
+            self.msn_process.kill()
+
     def stop_processes(self):
         """
         Stops all running processes.
@@ -310,24 +413,135 @@ class _ProcessManager:
             self.start_scheduler()
         self.start_workers()
 
+    def kill_all_workers(self):
+        """
+        Kill all running worker processes.
+
+        Args:
+            NA.
+        """
+        for p in self.cgn_processes:
+            if p.poll() is None:
+                p.kill()
+        self.cgn_processes.clear()
+
+        for p in self.tail_cgn_processes:
+            if p is not None:
+                p.kill()
+        self.tail_cgn_processes.clear()
+
+    def kill_single_worker(self, pid):
+        """
+        Kill one worker process with specified pid.
+
+        Args:
+            pid: Worker process' pid.
+        """
+        kill_status = False
+        for i in range(len(self.cgn_processes)):
+            p = self.cgn_processes[i]
+            if p.pid == pid and p.poll() is None:
+                p.kill()
+                del self.cgn_processes[i]
+                tail_p = self.tail_cgn_processes[i]
+                if tail_p is not None:
+                    tail_p.kill()
+                del self.tail_cgn_processes[i]
+                kill_status = True
+                break
+        if not kill_status:
+            logger.warning(f"There's no active worker with pid: {pid}")
+
+    def kill_workers(self, pids):
+        """
+        Kill worker process according to pids. Worker process with pid within pids list will be killed.
+
+        Args:
+            pids(list): a list of worker process pid. When local_ranks pids -1, kill all worker process.
+        """
+        if -1 in pids:
+            self.kill_all_workers()
+        else:
+            for pid in pids:
+                self.kill_single_worker(pid)
+        return 0
+
+    def monitor_rank_status(self, local_ranks):
+        """
+        Monitor the status of workers whose rank is within local_ranks list.
+
+        Args:
+            local_ranks(list): a list of local worker ranks. When local_ranks contains -1,
+                monitor all workers' status.
+        """
+        rank_status = {}
+        if -1 in local_ranks:
+            local_ranks = list(range(self.local_worker_num))
+        for i in local_ranks:
+            single_status = self.monitor_single_rank(i)
+            if single_status:
+                rank_status[i] = single_status
+        return rank_status
+
+    def monitor_single_rank(self, rank_id):
+        """
+        Monitor the status of a single worker with rank_id
+
+        Args:
+            rank_id: worker process's local rank, which is also device_id.
+        """
+        if 0 <= rank_id < self.local_worker_num:
+            global_rank_id = rank_id
+            if self.node_rank >= 0:
+                global_rank_id = self.node_rank * self.local_worker_num + rank_id
+            try:
+                p = self.proc_rank_map[rank_id]
+                p_status = p.poll()
+                if (not psutil.pid_exists(p.pid)) and (p_status != 0):
+                    p_status = 300
+                return {"pid": p.pid, "status": p_status, "global_rank": global_rank_id}
+            except KeyError:
+                logger.info(f"Process rank {rank_id} has not been initialized.")
+                return {"pid": None, "status": 200, "global_rank": global_rank_id}
+        else:
+            logger.warning(f"Invalid rank id!")
+        return {}
+
+    def start_all_workers(self):
+        """
+        Start all worker processes after killing all workers.
+
+        Args:
+            NA.
+        """
+        if self.cgn_processes:
+            self.kill_all_workers()
+        self.start_workers()
+        worker_status = self.monitor_rank_status([-1])
+        for i in range(self.local_worker_num):
+            if worker_status[i]["status"] != None:  # pylint: disable=singleton-comparison
+                return 1
+        return 0
+
     def _get_node_id_and_log_path(self, index):
         """
         Generate node id and log path for corresponding process.
         """
+        formatted_log_name = self.format_worker_log_name()
         if self.local_worker_num > self.worker_num:
             raise ValueError(f"Total worker number is {self.worker_num}, "
                              f"but got exceeded local worker number: {self.local_worker_num}.")
         if self.local_worker_num == self.worker_num:
-            return index, os.path.join(self.log_dir, "worker_" + str(index) + ".log")
+            return index, os.path.join(self.log_dir, formatted_log_name + "_" + str(index) + ".log")
 
         if self.node_rank >= 0:
             # We assume that each node has same process number.
             node_id = self.node_rank * self.local_worker_num + index
-            log_name = os.path.join(self.log_dir, "worker_" + str(node_id) + ".log")
+            log_name = os.path.join(self.log_dir, formatted_log_name + "_" + str(node_id) + ".log")
         else:
             # If node_rank is default value -1, let MindSpore assign rank id.
             node_id = None
-            log_name = os.path.join(self.log_dir, "worker_" + str(index) + ".log")
+            log_name = os.path.join(self.log_dir, formatted_log_name + "_" + str(index) + ".log")
         return node_id, log_name
 
     def _analyze_log(self):
@@ -350,3 +564,15 @@ class _ProcessManager:
             logger.error(f"Time out nodes are {time_out_node_ids}")
 
         os.system(f"grep -rn -E 'ERROR|CRITICAL|Traceback|Error' -C 5 {self.log_dir}")
+
+    def format_worker_log_name(self):
+        """
+        Format worker log files' name.
+        """
+        if not self.worker_log_name:
+            formatted_worker_log_name = "worker"
+        else:
+            current_ip = _get_local_ip(self.master_addr)
+            formatted_worker_log_name = re.sub(r'\{ip\}', current_ip, self.worker_log_name)
+            formatted_worker_log_name = re.sub(r'\{hostname\}', socket.gethostname(), formatted_worker_log_name)
+        return formatted_worker_log_name

mindspore/parallel/cluster/process_entity/_utils.py

@@ -16,8 +16,11 @@
 import os
 import json
 import socket
+import ipaddress
 import mindspore.log as logger
 
+CURRENT_IP = None
+
 def _generate_cmd(cmd, cmd_args, output_name):
     """
     Generates a command string to execute a Python script in the background, r
@@ -67,6 +70,24 @@ def _generate_url(addr, port):
     return url
 
 
+def _get_local_ip(ip_address):
+    """
+    Get current IP address.
+
+    """
+    global CURRENT_IP
+    if CURRENT_IP is None:
+        try:
+            s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+            s.connect((ip_address, 0))
+            CURRENT_IP = s.getsockname()[0]
+            s.close()
+        except Exception as e:
+            raise RuntimeError(f"Get local ip failed: {e}. Please check whether an accessible address "
+                               "is input by '--master_address'.")
+    return CURRENT_IP
+
+
 def _is_local_ip(ip_address):
     """
     Check if the current input IP address is a local IP address.
@@ -75,13 +96,8 @@ def _is_local_ip(ip_address):
     p = os.popen("ip -j addr")
     addr_info_str = p.read()
     p.close()
+    current_ip = _get_local_ip(ip_address)
     if not addr_info_str:
-        # This means this host has no "ip -j addr" command.
-        # We use socket module to get local ip address.
-        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-        s.connect((ip_address, 0))
-        current_ip = s.getsockname()[0]
-        s.close()
         return current_ip == ip_address
 
     addr_infos = json.loads(addr_info_str)
@@ -93,6 +109,25 @@ def _is_local_ip(ip_address):
     return False
 
 
+def _convert_addr_to_ip(master_addr):
+    """
+    Check whether the input parameter 'master_addr' is IPv4. If a hostname is inserted, it will be converted
+    to IP and then set as master host's IP.
+
+    """
+    try:
+        ipaddress.IPv4Address(master_addr)
+        return master_addr
+    except ipaddress.AddressValueError:
+        try:
+            ip_address = socket.gethostbyname(master_addr)
+            logger.info(f"Convert input host name:{master_addr} to ip address:{ip_address}.")
+            return ip_address
+        except socket.gaierror as e:
+            raise RuntimeError(f"DNS resolution failed: {e}. Please check whether a correct host name "
+                               "is input by '--master_address'.")
+
+
 def _send_scale_num(url, scale_num):
     """
     Send an HTTP request to a specified URL, informing scale_num.

mindspore/parallel/cluster/run.py

@@ -37,8 +37,8 @@ def get_args():
     parser.add_argument(
         "--master_addr",
         default="127.0.0.1", type=str,
-        help="specifies the IP address of the scheduler and its data type is string."
-        " Allowed values: valid IP addresses."
+        help="specifies the IP address or the host name of the scheduler and its data type is string."
+        " Allowed values: valid IP addresses or valid host name."
     )
     parser.add_argument(
         "--master_port", default=8118, type=int,
@@ -85,13 +85,13 @@ def get_args():
         "--sim_level",
         default=-1,
         type=int,
-        choices=[0, 1],
+        choices=[0, 1, 2, 3],
         help="specifies simulation level. When this argument is set, msrun only spawns one process "
              "but export RANK_SIZE with value worker_num and RANK_ID with value sim_rank_id."
     )
     parser.add_argument(
         "--sim_rank_id",
-        default=0,
+        default=-1,
         type=int,
         help="specifies simulation process's rank id. Only one process is spawned in simulation scenario."
     )
@@ -102,6 +102,23 @@ def get_args():
         help="specifies rank table file path. This path is not used to initialize distributed job in "
              "'rank table file manner' but to help support other features."
     )
+    parser.add_argument(
+        "--worker_log_name",
+        default="",
+        type=str,
+        help="Specifies the worker log file name as a string for current node; the default is worker_[rankid]. "
+             "Support configuring the current IP address and host name by using {ip} and {hostname} respectively. "
+             "e.g. --worker_log_name=worker_{ip}_{hostname}_test, worker [rankid] log name for current node "
+             "will be worker_[real IP address]_[real host name]_test_[rankid]."
+    )
+    parser.add_argument(
+        "--tail_worker_log",
+        default="-1",
+        type=str,
+        help="Only tail worker log to console when '--join=True' and the configured value should be within "
+             "[0, local_worker_num], otherwise worker log will not be tail. All worker logs will be tail by "
+             "default. Support tail the specified worker log (e.g. --tail_log=0 tail the worker 0 log to console)."
+    )
     parser.add_argument(
         "task_script",
         type=str,

mindspore/parallel/function/__init__.py

@@ -0,0 +1,24 @@
+# Copyright 2025 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+"""
+Parallel function operator
+"""
+
+from mindspore.parallel.function.reshard_func import reshard
+
+__all__ = []
+__all__.extend(reshard_func.__all__)
+__all__.sort()