mindspore 2.4.10__cp310-cp310-manylinux1_x86_64.whl → 2.5.0__cp310-cp310-manylinux1_x86_64.whl

mindspore/parallel/cluster/process_entity/_api.py

@@ -16,23 +16,30 @@
 import os
 import re
 import sys
+import signal
 import subprocess
+import socket
 import mindspore.log as logger
 from ._utils import _generate_cmd_args_list, _generate_cmd_args_list_with_core, _generate_url,\
-                    _is_local_ip, _send_scale_num
+                    _is_local_ip, _convert_addr_to_ip, _send_scale_num, _get_local_ip
 
 class _Node:
     """
     Base class for dynamic networking nodes.
 
     """
-    def __init__(self, worker_num, sched_host, sched_port, timeout, args_list, output_file):
+    def __init__(self, worker_num, sched_host, sched_port, timeout, args_list, output_file, tail_worker_log,
+                 join, is_simulation):
         self.worker_num = worker_num
         self.sched_host = sched_host
         self.sched_port = sched_port
         self.args_list = args_list
         self.output_file = output_file
         self.timeout = timeout
+        self.tail_worker_log = tail_worker_log
+        self.join = join
+        self.is_simulation = is_simulation
+
 
     def run(self):
         """
@@ -40,9 +47,11 @@ class _Node:
 
         """
         os.environ["MS_WORKER_NUM"] = str(self.worker_num)
-        os.environ["MS_SCHED_HOST"] = self.sched_host
-        os.environ["MS_SCHED_PORT"] = str(self.sched_port)
-        os.environ["MS_TOPO_TIMEOUT"] = str(self.timeout)
+        # If simulation level is set, environment variables for dynamic networking will not be set and scheduler will not be started.
+        if not self.is_simulation:
+            os.environ["MS_SCHED_HOST"] = self.sched_host
+            os.environ["MS_SCHED_PORT"] = str(self.sched_port)
+            os.environ["MS_TOPO_TIMEOUT"] = str(self.timeout)
 
 class _MetaServerNode(_Node):
     """
@@ -63,8 +72,10 @@ class _ComputeGraphNode(_Node):
     """
     Worker node for dynamic networking. Inherits from the Node class.
     """
-    def __init__(self, worker_num, sched_host, sched_port, timeout, node_id, args_list, output_file):
-        super().__init__(worker_num, sched_host, sched_port, timeout, args_list, output_file)
+    def __init__(self, worker_num, sched_host, sched_port, timeout, node_id, args_list, output_file,
+                 tail_worker_log, join, is_simulation):
+        super().__init__(worker_num, sched_host, sched_port, timeout, args_list, output_file,
+                         tail_worker_log, join, is_simulation)
         self.node_id = node_id
 
 
@@ -78,9 +89,36 @@ class _ComputeGraphNode(_Node):
         super().run()
         if self.node_id is not None:
             os.environ["MS_NODE_ID"] = str(self.node_id)
-        os.environ["MS_ROLE"] = "MS_WORKER"
+        # If simulation level is set, environment variable 'MS_ROLE' will not be set.
+        if not self.is_simulation:
+            os.environ["MS_ROLE"] = "MS_WORKER"
+        tail_worker_process = None
+        is_tail_worker_log = self.enable_tail_worker_log()
+        if self.join and not is_tail_worker_log:
+            logger.warning(f"The '--tail_worker_log' is:{self.tail_worker_log}, "
+                           f"which doesn't contain this worker {self.node_id}."
+                           f" So this worker {self.node_id}'s log will not be output to console. Reset "
+                           "'--tail_worker_log', if you want to output this worker's log to console.")
         with open(self.output_file, "w") as file_handle:
-            return subprocess.Popen(self.args_list, stdout=file_handle, stderr=subprocess.STDOUT)
+            worker_process = subprocess.Popen(self.args_list, preexec_fn=os.setsid, stdout=file_handle,
+                                              stderr=subprocess.STDOUT)
+            if self.join and is_tail_worker_log:
+                tail_worker_process = self.output_to_console()
+            return worker_process, tail_worker_process
+
+    def output_to_console(self):
+        """
+        Output worker log file to console.
+        """
+        return subprocess.Popen(['/usr/bin/tail', '-f', self.output_file])
+
+    def enable_tail_worker_log(self):
+        tail_worker_log_list = []
+        if self.tail_worker_log != "-1":
+            tail_worker_log_list.extend([int(num) for num in self.tail_worker_log.split(',')])
+        if self.tail_worker_log != "-1" and self.node_id not in tail_worker_log_list:
+            return False
+        return True
 
 
 class _ProcessManager:
@@ -99,13 +137,14 @@ class _ProcessManager:
         """
         self.msn_process = None
         self.cgn_processes = []
+        self.tail_cgn_processes = []
 
-        """`is_master` flags whether the current node is the master node."""
-        self.is_master = _is_local_ip(args.master_addr)
-
-        self.master_addr = args.master_addr
+        self.master_addr = _convert_addr_to_ip(args.master_addr)
         self.master_port = args.master_port
 
+        """`is_master` flags whether the current node is the master node."""
+        self.is_master = _is_local_ip(self.master_addr)
+
         self.worker_num = args.worker_num
         if self.worker_num <= 0:
             raise ValueError(f"worker_num must be greater than 0, but got {self.worker_num}.")
@@ -115,6 +154,8 @@ class _ProcessManager:
 
         self.log_dir = args.log_dir
         self.join = args.join
+        self.worker_log_name = args.worker_log_name
+        self.tail_worker_log = args.tail_worker_log
         self.cluster_time_out = args.cluster_time_out
         self.bind_core = args.bind_core
         self.rank_table_file = args.rank_table_file
@@ -123,19 +164,21 @@ class _ProcessManager:
         self.sim_rank_id = args.sim_rank_id
         self.is_simulation = (self.sim_level != -1)
         if self.is_simulation:
-            # If simulation level is set, reset the worker_num and local_worker_num to 1
-            # so that host cluster could be initialized.
-            self.worker_num = 1
-            self.local_worker_num = 1
             os.environ["MS_SIMULATION_LEVEL"] = str(self.sim_level)
         elif os.getenv("MS_SIMULATION_LEVEL"):
-            # If simulation level env is set, load RANK_ID and RANK_SIZE envs.
-            self.worker_num = 1
-            self.local_worker_num = 1
             self.is_simulation = True
-            self.sim_rank_id = os.getenv("RANK_ID", "0")
+            self.sim_rank_id = int(os.getenv("RANK_ID", "-1"))
             if os.getenv("RANK_SIZE"):
                 self.exported_rank_size = os.getenv("RANK_SIZE")
+        # If sim_rank_id is set, single worker can be started.
+        if self.is_simulation and (self.sim_rank_id != -1):
+            logger.info(f"Simulation rank id is set to {self.sim_rank_id}, will dryrun a single process.")
+            self.local_worker_num = 1
+        if self.is_simulation and self.local_worker_num > 128:
+            self.local_worker_num = 1
+            self.sim_rank_id = 0
+            logger.warning(f"In dryrun case, local worker num is set to larger than 128. "
+                           "To avoid a system clash, local worker num is set to 1.")
 
         self.cmd = args.task_script
         self.cmd_args = args.task_script_args
@@ -173,7 +216,7 @@ class _ProcessManager:
             else:
                 sys.exit()
         else:
-            if self.is_master:
+            if self.is_master and not self.is_simulation:
                 self.start_scheduler()
         self.start_workers()
 
@@ -190,7 +233,8 @@ class _ProcessManager:
         os.environ['RANK_ID'] = str(0)
         msn = _MetaServerNode(self.worker_num, self.master_addr, self.master_port, self.cluster_time_out,
                               _generate_cmd_args_list(self.cmd, self.cmd_args),
-                              os.path.join(self.log_dir, "scheduler.log"))
+                              os.path.join(self.log_dir, "scheduler.log"), self.tail_worker_log, self.join,
+                              self.is_simulation)
         self.msn_process = msn.run()
 
     def start_workers(self):
@@ -208,9 +252,6 @@ class _ProcessManager:
                            "You can access 'RANK_ID' environment variable after calling "
                            "'mindspore.communication.init()'")
 
-        if self.is_simulation and self.worker_num != 1:
-            raise ValueError(f"Simulation level is set, worker_num must be 1, but got {self.worker_num}.")
-
         for i in range(self.local_worker_num):
             os.environ["DEVICE_ID"] = str(i)
             node_id, log_name = self._get_node_id_and_log_path(i)
@@ -223,9 +264,10 @@ class _ProcessManager:
                 os.environ["RANK_ID"] = str(node_id)
                 logger.warning(f"Start worker process with rank id:{node_id}, log file:{log_name}. "
                                "Environment variable [RANK_ID] is exported.")
-            if self.is_simulation:
-                # Reset RANK_ID env to sim_rank_id.
+            if self.is_simulation and (self.sim_rank_id != -1):
+                # Reset RANK_ID env to sim_rank_id if sim_rank_id is set.
                 os.environ["RANK_ID"] = str(self.sim_rank_id)
+                logger.warning(f"In dryrun case, RANK_ID is assigned to {self.sim_rank_id}.")
 
             cpu_num = subprocess.getoutput("cat /proc/cpuinfo|grep processor|wc -l")
             if not cpu_num.isdigit():
@@ -238,9 +280,11 @@ class _ProcessManager:
             else:
                 cmd = _generate_cmd_args_list(self.cmd, self.cmd_args)
             cgn = _ComputeGraphNode(self.worker_num, self.master_addr, self.master_port, self.cluster_time_out,
-                                    node_id, cmd, log_name)
-            process = cgn.run()
+                                    node_id, cmd, log_name, self.tail_worker_log, self.join, self.is_simulation)
+            process, tail_process = cgn.run()
             self.cgn_processes.append(process)
+            self.tail_cgn_processes.append(tail_process)
+
 
     def join_processes(self):
         """
@@ -248,8 +292,14 @@ class _ProcessManager:
         If there's any process does not exit normally, logs will be analyzed
         so that understandable root cause of exception could be returned.
         """
+        def signal_handler(sig, frame):
+            logger.warning("msrun process received SIGNIN (Ctrl+C), terminating all workers.")
+            self.kill_all_processes()
+            sys.exit(0)
+
         has_exception = False
         success_cgn_processes = set()
+        signal.signal(signal.SIGINT, signal_handler)
         while True:
             # Traversal all workers and kill immediately if any exception happens.
             for p in self.cgn_processes:
@@ -266,15 +316,14 @@ class _ProcessManager:
 
             if has_exception:
                 logger.warning("There's worker exits with exception, kill all other workers.")
-                for p in self.cgn_processes:
-                    if p.poll() is None:
-                        p.kill()
+                self.kill_worker_processes()
+                self.kill_tail_log_processes()
                 break
             elif len(success_cgn_processes) == len(self.cgn_processes):
                 logger.info("All workers successfully exit!")
+                self.kill_tail_log_processes()
                 break
 
-
         if self.msn_process:
             self.msn_process.wait()
             if self.msn_process.returncode != 0:
@@ -287,6 +336,35 @@ class _ProcessManager:
             raise RuntimeError("Distributed job exited with exception. Please check logs in "
                                f"directory: {self.log_dir}.")
 
+    def kill_tail_log_processes(self):
+        """
+        Kills all tail worker log processes.
+
+        """
+        for p_tail in self.tail_cgn_processes:
+            if p_tail is not None:
+                logger.debug("Tail worker log process:{p_tail.pid} has been killed!")
+                p_tail.kill()
+
+    def kill_worker_processes(self):
+        """
+        Kills all worker processes.
+
+        """
+        for p in self.cgn_processes:
+            if p.poll() is None:
+                os.killpg(os.getpgid(p.pid), signal.SIGKILL)
+
+    def kill_all_processes(self):
+        """
+        Kills all running processes, including scheduler, worker and tail log.
+
+        """
+        self.kill_worker_processes()
+        self.kill_tail_log_processes()
+        if self.msn_process.poll() is None:
+            self.msn_process.kill()
+
     def stop_processes(self):
         """
         Stops all running processes.
@@ -310,26 +388,29 @@ class _ProcessManager:
             self.start_scheduler()
         self.start_workers()
 
+
     def _get_node_id_and_log_path(self, index):
         """
         Generate node id and log path for corresponding process.
         """
+        formatted_log_name = self.format_worker_log_name()
         if self.local_worker_num > self.worker_num:
             raise ValueError(f"Total worker number is {self.worker_num}, "
                              f"but got exceeded local worker number: {self.local_worker_num}.")
         if self.local_worker_num == self.worker_num:
-            return index, os.path.join(self.log_dir, "worker_" + str(index) + ".log")
+            return index, os.path.join(self.log_dir, formatted_log_name + "_" + str(index) + ".log")
 
         if self.node_rank >= 0:
             # We assume that each node has same process number.
             node_id = self.node_rank * self.local_worker_num + index
-            log_name = os.path.join(self.log_dir, "worker_" + str(node_id) + ".log")
+            log_name = os.path.join(self.log_dir, formatted_log_name + "_" + str(node_id) + ".log")
         else:
             # If node_rank is default value -1, let MindSpore assign rank id.
             node_id = None
-            log_name = os.path.join(self.log_dir, "worker_" + str(index) + ".log")
+            log_name = os.path.join(self.log_dir, formatted_log_name + "_" + str(index) + ".log")
         return node_id, log_name
 
+
     def _analyze_log(self):
         """
         Analyze exception logs.
@@ -350,3 +431,16 @@ class _ProcessManager:
             logger.error(f"Time out nodes are {time_out_node_ids}")
 
         os.system(f"grep -rn -E 'ERROR|CRITICAL|Traceback|Error' -C 5 {self.log_dir}")
+
+
+    def format_worker_log_name(self):
+        """
+        Format worker log files' name.
+        """
+        if not self.worker_log_name:
+            formatted_worker_log_name = "worker"
+        else:
+            current_ip = _get_local_ip(self.master_addr)
+            formatted_worker_log_name = re.sub(r'\{ip\}', current_ip, self.worker_log_name)
+            formatted_worker_log_name = re.sub(r'\{hostname\}', socket.gethostname(), formatted_worker_log_name)
+        return formatted_worker_log_name

mindspore/parallel/cluster/process_entity/_utils.py

@@ -16,8 +16,11 @@
 import os
 import json
 import socket
+import ipaddress
 import mindspore.log as logger
 
+CURRENT_IP = None
+
 def _generate_cmd(cmd, cmd_args, output_name):
     """
     Generates a command string to execute a Python script in the background, r
@@ -67,6 +70,24 @@ def _generate_url(addr, port):
     return url
 
 
+def _get_local_ip(ip_address):
+    """
+    Get current IP address.
+
+    """
+    global CURRENT_IP
+    if CURRENT_IP is None:
+        try:
+            s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+            s.connect((ip_address, 0))
+            CURRENT_IP = s.getsockname()[0]
+            s.close()
+        except Exception as e:
+            raise RuntimeError(f"Get local ip failed: {e}. Please check whether an accessible address "
+                               "is input by '--master_address'.")
+    return CURRENT_IP
+
+
 def _is_local_ip(ip_address):
     """
     Check if the current input IP address is a local IP address.
@@ -75,13 +96,8 @@ def _is_local_ip(ip_address):
     p = os.popen("ip -j addr")
     addr_info_str = p.read()
     p.close()
+    current_ip = _get_local_ip(ip_address)
     if not addr_info_str:
-        # This means this host has no "ip -j addr" command.
-        # We use socket module to get local ip address.
-        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-        s.connect((ip_address, 0))
-        current_ip = s.getsockname()[0]
-        s.close()
         return current_ip == ip_address
 
     addr_infos = json.loads(addr_info_str)
@@ -93,6 +109,25 @@ def _is_local_ip(ip_address):
     return False
 
 
+def _convert_addr_to_ip(master_addr):
+    """
+    Check whether the input parameter 'master_addr' is IPv4. If a hostname is inserted, it will be converted
+    to IP and then set as master host's IP.
+
+    """
+    try:
+        ipaddress.IPv4Address(master_addr)
+        return master_addr
+    except ipaddress.AddressValueError:
+        try:
+            ip_address = socket.gethostbyname(master_addr)
+            logger.info(f"Convert input host name:{master_addr} to ip address:{ip_address}.")
+            return ip_address
+        except socket.gaierror as e:
+            raise RuntimeError(f"DNS resolution failed: {e}. Please check whether a correct host name "
+                               "is input by '--master_address'.")
+
+
 def _send_scale_num(url, scale_num):
     """
     Send an HTTP request to a specified URL, informing scale_num.

mindspore/parallel/cluster/run.py

@@ -37,8 +37,8 @@ def get_args():
     parser.add_argument(
         "--master_addr",
         default="127.0.0.1", type=str,
-        help="specifies the IP address of the scheduler and its data type is string."
-        " Allowed values: valid IP addresses."
+        help="specifies the IP address or the host name of the scheduler and its data type is string."
+        " Allowed values: valid IP addresses or valid host name."
     )
     parser.add_argument(
         "--master_port", default=8118, type=int,
@@ -91,7 +91,7 @@ def get_args():
     )
     parser.add_argument(
         "--sim_rank_id",
-        default=0,
+        default=-1,
         type=int,
         help="specifies simulation process's rank id. Only one process is spawned in simulation scenario."
     )
@@ -102,6 +102,23 @@ def get_args():
         help="specifies rank table file path. This path is not used to initialize distributed job in "
              "'rank table file manner' but to help support other features."
     )
+    parser.add_argument(
+        "--worker_log_name",
+        default="",
+        type=str,
+        help="Specifies the worker log file name as a string for current node; the default is worker_[rankid]. "
+             "Support configuring the current IP address and host name by using {ip} and {hostname} respectively. "
+             "e.g. --worker_log_name=worker_{ip}_{hostname}_test, worker [rankid] log name for current node "
+             "will be worker_[real IP address]_[real host name]_test_[rankid]."
+    )
+    parser.add_argument(
+        "--tail_worker_log",
+        default="-1",
+        type=str,
+        help="Only tail worker log to console when '--join=True' and the configured value should be within "
+             "[0, local_worker_num], otherwise worker log will not be tail. All worker logs will be tail by "
+             "default. Support tail the specified worker log (e.g. --tail_log=0 tail the worker 0 log to console)."
+    )
     parser.add_argument(
         "task_script",
         type=str,

mindspore/parallel/parameter_broadcast.py

@@ -56,7 +56,7 @@ def parameter_broadcast(net, layout, cur_rank=0, initial_rank=0):
         >>> from mindspore.parallel.parameter_broadcast import parameter_broadcast
         >>> from mindspore.train.serialization import load_checkpoint, load_param_into_net
         >>> ms.set_context(mode=ms.GRAPH_MODE)
-        >>> ms.set_context(max_device_memory="28GB")
+        >>> ms.runtime.set_memory(max_size="28GB")
         >>> ms.set_auto_parallel_context(parallel_mode=ms.ParallelMode.SEMI_AUTO_PARALLEL)
         >>> init()
         >>> ms.set_seed(1)

mindspore/parallel/shard.py

@@ -24,6 +24,9 @@ class Layout:
     """
     Parallel layout describes the detailed sharding information.
 
+    For more detailed information, refer to the file `Higher-order Operator-level Parallelism
+    <https://www.mindspore.cn/docs/en/master/model_train/parallel/advanced_operator_parallel.html>`_.
+
     Note:
         - It is valid only in semi auto parallel or auto parallel mode.
         - The multiplication result of the `device_matrix` must be equal to the device count in a pipeline stage.