skypilot-nightly 1.0.0.dev20251203__py3-none-any.whl → 1.0.0.dev20251210__py3-none-any.whl

sky/__init__.py

@@ -7,7 +7,7 @@ import urllib.request
 from sky.utils import directory_utils
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '3ff39aba6d4752d5c3b09e3fa7d778cefea39370'
+_SKYPILOT_COMMIT_SHA = 'c28d94abd967c1a7494e3c343f92eb6d02d29541'
 
 
 def _get_git_commit():
@@ -37,7 +37,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20251203'
+__version__ = '1.0.0.dev20251210'
 __root_dir__ = directory_utils.get_sky_dir()
 
 
@@ -140,6 +140,7 @@ Cudo = clouds.Cudo
 GCP = clouds.GCP
 Lambda = clouds.Lambda
 SCP = clouds.SCP
+Slurm = clouds.Slurm
 Kubernetes = clouds.Kubernetes
 K8s = Kubernetes
 OCI = clouds.OCI
@@ -170,6 +171,7 @@ __all__ = [
     'RunPod',
     'Vast',
     'SCP',
+    'Slurm',
     'Vsphere',
     'Fluidstack',
     'Nebius',

sky/adaptors/aws.py

@@ -28,7 +28,6 @@ This is informed by the following boto3 docs:
 
 # pylint: disable=import-outside-toplevel
 
-import functools
 import logging
 import threading
 import time
@@ -69,65 +68,6 @@ version = 1
 _MAX_ATTEMPT_FOR_CREATION = 5
 
 
-class _ThreadLocalTTLCache(threading.local):
-    """Thread-local storage for _thread_local_lru_cache decorator."""
-
-    def __init__(self, func, maxsize: int, ttl: int):
-        super().__init__()
-        self.func = func
-        self.maxsize = maxsize
-        self.ttl = ttl
-
-    def get_cache(self):
-        if not hasattr(self, 'cache'):
-            self.cache = annotations.ttl_cache(scope='request',
-                                               maxsize=self.maxsize,
-                                               ttl=self.ttl,
-                                               timer=time.time)(self.func)
-        return self.cache
-
-
-def _thread_local_ttl_cache(maxsize=32, ttl=60 * 55):
-    """Thread-local TTL cache decorator.
-
-    Args:
-        maxsize: Maximum size of the cache.
-        ttl: Time to live for the cache in seconds.
-             Default is 55 minutes, a bit less than 1 hour
-             default lifetime of an STS token.
-    """
-
-    def decorator(func):
-        # Create thread-local storage for the LRU cache
-        local_cache = _ThreadLocalTTLCache(func, maxsize, ttl)
-
-        # We can't apply the lru_cache here, because this runs at import time
-        # so we will always have the main thread's cache.
-
-        @functools.wraps(func)
-        def wrapper(*args, **kwargs):
-            # We are within the actual function call, which may be on a thread,
-            # so local_cache.cache will return the correct thread-local cache,
-            # which we can now apply and immediately call.
-            return local_cache.get_cache()(*args, **kwargs)
-
-        def cache_info():
-            # Note that this will only give the cache info for the current
-            # thread's cache.
-            return local_cache.get_cache().cache_info()
-
-        def cache_clear():
-            # Note that this will only clear the cache for the current thread.
-            local_cache.get_cache().cache_clear()
-
-        wrapper.cache_info = cache_info  # type: ignore[attr-defined]
-        wrapper.cache_clear = cache_clear  # type: ignore[attr-defined]
-
-        return wrapper
-
-    return decorator
-
-
 def _assert_kwargs_builtin_type(kwargs):
     assert all(isinstance(v, (int, float, str)) for v in kwargs.values()), (
         f'kwargs should not contain none built-in types: {kwargs}')
@@ -174,7 +114,7 @@ def get_workspace_profile() -> Optional[str]:
 
 # The TTL cache needs to be thread-local to avoid multiple threads sharing the
 # same session object, which is not guaranteed to be thread-safe.
-@_thread_local_ttl_cache()
+@annotations.thread_local_ttl_cache()
 def session(check_credentials: bool = True, profile: Optional[str] = None):
     """Create an AWS session.
 

sky/adaptors/slurm.py

@@ -0,0 +1,478 @@
+"""Slurm adaptor for SkyPilot."""
+
+import logging
+import re
+import time
+from typing import Dict, List, NamedTuple, Optional, Tuple
+
+from sky.utils import command_runner
+from sky.utils import subprocess_utils
+from sky.utils import timeline
+
+logger = logging.getLogger(__name__)
+
+# ASCII Unit Separator (\x1f) to handle values with spaces
+# and other special characters.
+SEP = r'\x1f'
+
+# Regex pattern to extract partition names from scontrol output
+# Matches PartitionName=<name> and captures until the next field
+_PARTITION_NAME_REGEX = re.compile(r'PartitionName=(.+?)(?:\s+\w+=|$)')
+
+# Default timeout for waiting for job nodes to be allocated, in seconds.
+_SLURM_DEFAULT_PROVISION_TIMEOUT = 10
+
+
+class SlurmPartition(NamedTuple):
+    """Information about the Slurm partitions."""
+    name: str
+    is_default: bool
+
+
+# TODO(kevin): Add more API types for other client functions.
+class NodeInfo(NamedTuple):
+    """Information about a Slurm node from sinfo."""
+    node: str
+    state: str
+    gres: str
+    cpus: int
+    memory_gb: float
+    # The default partition contains a '*' at the end of the name.
+    # It is the caller's responsibility to strip the '*' if needed.
+    partition: str
+
+
+class SlurmClient:
+    """Client for Slurm control plane operations."""
+
+    def __init__(
+        self,
+        ssh_host: str,
+        ssh_port: int,
+        ssh_user: str,
+        ssh_key: Optional[str],
+        ssh_proxy_command: Optional[str] = None,
+    ):
+        """Initialize SlurmClient.
+
+        Args:
+            ssh_host: Hostname of the Slurm controller.
+            ssh_port: SSH port on the controller.
+            ssh_user: SSH username.
+            ssh_key: Path to SSH private key, or None for keyless SSH.
+            ssh_proxy_command: Optional SSH proxy command.
+        """
+        self.ssh_host = ssh_host
+        self.ssh_port = ssh_port
+        self.ssh_user = ssh_user
+        self.ssh_key = ssh_key
+        self.ssh_proxy_command = ssh_proxy_command
+
+        # Internal runner for executing Slurm CLI commands
+        # on the controller node.
+        self._runner = command_runner.SSHCommandRunner(
+            (ssh_host, ssh_port),
+            ssh_user,
+            ssh_key,
+            ssh_proxy_command=ssh_proxy_command,
+        )
+
+    def query_jobs(
+        self,
+        job_name: Optional[str] = None,
+        state_filters: Optional[List[str]] = None,
+    ) -> List[str]:
+        """Query Slurm jobs by state and optional name.
+
+        Args:
+            job_name: Optional job name to filter by.
+            state_filters: List of job states to filter by
+                (e.g., ['running', 'pending']). If None, returns all jobs.
+
+        Returns:
+            List of job IDs matching the filters.
+        """
+        cmd = 'squeue --me -h -o "%i"'
+        if state_filters is not None:
+            state_filters_str = ','.join(state_filters)
+            cmd += f' --states {state_filters_str}'
+        if job_name is not None:
+            cmd += f' --name {job_name}'
+
+        rc, stdout, stderr = self._runner.run(cmd,
+                                              require_outputs=True,
+                                              stream_logs=False)
+        subprocess_utils.handle_returncode(rc,
+                                           cmd,
+                                           'Failed to query Slurm jobs.',
+                                           stderr=stderr)
+
+        job_ids = stdout.strip().splitlines()
+        return job_ids
+
+    def cancel_jobs_by_name(self,
+                            job_name: str,
+                            signal: Optional[str] = None,
+                            full: bool = False) -> None:
+        """Cancel Slurm job(s) by name.
+
+        Args:
+            job_name: Name of the job(s) to cancel.
+            signal: Optional signal to send to the job(s).
+            full: If True, signals the batch script and its children processes.
+                By default, signals other than SIGKILL are not sent to the
+                batch step (the shell script).
+        """
+        cmd = f'scancel --name {job_name}'
+        if signal is not None:
+            cmd += f' --signal {signal}'
+        if full:
+            cmd += ' --full'
+        rc, stdout, stderr = self._runner.run(cmd,
+                                              require_outputs=True,
+                                              stream_logs=False)
+        subprocess_utils.handle_returncode(rc,
+                                           cmd,
+                                           f'Failed to cancel job {job_name}.',
+                                           stderr=stderr)
+        logger.debug(f'Successfully cancelled job {job_name}: {stdout}')
+
+    def info(self) -> str:
+        """Get Slurm cluster information.
+
+        This is useful for checking if the cluster is accessible and
+        retrieving node information.
+
+        Returns:
+            The stdout output from sinfo.
+        """
+        cmd = 'sinfo'
+        rc, stdout, stderr = self._runner.run(cmd,
+                                              require_outputs=True,
+                                              stream_logs=False)
+        subprocess_utils.handle_returncode(
+            rc, cmd, 'Failed to get Slurm cluster information.', stderr=stderr)
+        return stdout
+
+    def info_nodes(self) -> List[NodeInfo]:
+        """Get Slurm node information.
+
+        Returns node names, states, GRES (generic resources like GPUs),
+        CPUs, memory (MB), and partitions.
+        """
+        cmd = (f'sinfo -h --Node -o '
+               f'"%N{SEP}%t{SEP}%G{SEP}%c{SEP}%m{SEP}%P"')
+        rc, stdout, stderr = self._runner.run(cmd,
+                                              require_outputs=True,
+                                              stream_logs=False)
+        subprocess_utils.handle_returncode(
+            rc, cmd, 'Failed to get Slurm node information.', stderr=stderr)
+
+        nodes = []
+        for line in stdout.splitlines():
+            parts = line.split(SEP)
+            if len(parts) != 6:
+                raise RuntimeError(
+                    f'Unexpected output format from sinfo: {line!r}')
+            try:
+                node_info = NodeInfo(node=parts[0],
+                                     state=parts[1],
+                                     gres=parts[2],
+                                     cpus=int(parts[3]),
+                                     memory_gb=int(parts[4]) / 1024.0,
+                                     partition=parts[5])
+                nodes.append(node_info)
+            except ValueError as e:
+                raise RuntimeError(
+                    f'Failed to parse node info from line: {line!r}. '
+                    f'Error: {e}') from e
+
+        return nodes
+
+    def node_details(self, node_name: str) -> Dict[str, str]:
+        """Get detailed Slurm node information.
+
+        Returns:
+            A dictionary of node attributes.
+        """
+
+        def _parse_scontrol_node_output(output: str) -> Dict[str, str]:
+            """Parses the key=value output of 'scontrol show node'."""
+            node_info = {}
+            # Split by space, handling values that might have spaces
+            # if quoted. This is simplified; scontrol can be complex.
+            parts = output.split()
+            for part in parts:
+                if '=' in part:
+                    key, value = part.split('=', 1)
+                    # Simple quote removal, might need refinement
+                    value = value.strip('\'"')
+                    node_info[key] = value
+            return node_info
+
+        cmd = f'scontrol show node {node_name}'
+        rc, node_details, _ = self._runner.run(cmd,
+                                               require_outputs=True,
+                                               stream_logs=False)
+        subprocess_utils.handle_returncode(
+            rc,
+            cmd,
+            f'Failed to get detailed node information for {node_name}.',
+            stderr=node_details)
+        node_info = _parse_scontrol_node_output(node_details)
+        return node_info
+
+    def get_node_jobs(self, node_name: str) -> List[str]:
+        """Get the list of jobs for a given node name.
+
+        Returns:
+            A list of job names for the current user on the node.
+        """
+        cmd = f'squeue --me -h --nodelist {node_name} -o "%b"'
+        rc, stdout, stderr = self._runner.run(cmd,
+                                              require_outputs=True,
+                                              stream_logs=False)
+        subprocess_utils.handle_returncode(
+            rc, cmd, f'Failed to get jobs for node {node_name}.', stderr=stderr)
+        return stdout.splitlines()
+
+    def get_job_state(self, job_id: str) -> Optional[str]:
+        """Get the state of a Slurm job.
+
+        Args:
+            job_id: The Slurm job ID.
+
+        Returns:
+            The job state (e.g., 'PENDING', 'RUNNING', 'COMPLETED', etc.),
+            or None if the job is not found.
+        """
+        # Use --only-job-state since we only need the job state.
+        # This reduces the work required by slurmctld.
+        cmd = f'squeue -h --only-job-state --jobs {job_id} -o "%T"'
+        rc, stdout, stderr = self._runner.run(cmd,
+                                              require_outputs=True,
+                                              stream_logs=False)
+        if rc != 0:
+            # Job may not exist
+            logger.debug(f'Failed to get job state for job {job_id}: {stderr}')
+            return None
+
+        state = stdout.strip()
+        return state if state else None
+
+    @timeline.event
+    def get_job_reason(self, job_id: str) -> Optional[str]:
+        """Get the reason a job is in its current state
+
+        Args:
+            job_id: The Slurm job ID.
+        """
+        # Without --states all, squeue omits terminated jobs.
+        cmd = f'squeue -h --jobs {job_id} --states all -o "%r"'
+        rc, stdout, stderr = self._runner.run(cmd,
+                                              require_outputs=True,
+                                              stream_logs=False)
+        if rc != 0:
+            logger.debug(f'Failed to get job info for job {job_id}: {stderr}')
+            return None
+
+        output = stdout.strip()
+        if not output:
+            return None
+
+        return output if output != 'None' else None
+
+    @timeline.event
+    def wait_for_job_nodes(self, job_id: str, timeout: int) -> None:
+        """Wait for a Slurm job to have nodes allocated.
+
+        Args:
+            job_id: The Slurm job ID.
+            timeout: Maximum time to wait in seconds.
+        """
+        start_time = time.time()
+        last_state = None
+
+        while time.time() - start_time < timeout:
+            state = self.get_job_state(job_id)
+
+            if state != last_state:
+                logger.debug(f'Job {job_id} state: {state}')
+                last_state = state
+
+            if state is None:
+                raise RuntimeError(f'Job {job_id} not found. It may have been '
+                                   'cancelled or failed.')
+
+            if state in ('COMPLETED', 'CANCELLED', 'FAILED', 'TIMEOUT'):
+                raise RuntimeError(
+                    f'Job {job_id} terminated with state {state} '
+                    'before nodes were allocated.')
+            # TODO(kevin): Log reason for pending.
+
+            # Check if nodes are allocated by trying to get node list
+            cmd = f'squeue -h --jobs {job_id} -o "%N"'
+            rc, stdout, stderr = self._runner.run(cmd,
+                                                  require_outputs=True,
+                                                  stream_logs=False)
+
+            if rc == 0 and stdout.strip():
+                # Nodes are allocated
+                logger.debug(
+                    f'Job {job_id} has nodes allocated: {stdout.strip()}')
+                return
+            elif rc != 0:
+                logger.debug(f'Failed to get nodes for job {job_id}: {stderr}')
+
+            # Wait before checking again
+            time.sleep(2)
+
+        raise TimeoutError(f'Job {job_id} did not get nodes allocated within '
+                           f'{timeout} seconds. Last state: {last_state}')
+
+    @timeline.event
+    def get_job_nodes(
+            self,
+            job_id: str,
+            wait: bool = True,
+            timeout: Optional[int] = None) -> Tuple[List[str], List[str]]:
+        """Get the list of nodes and their IPs for a given job ID.
+
+        The ordering is guaranteed to be stable for the lifetime of the job.
+
+        Args:
+            job_id: The Slurm job ID.
+            wait: If True, wait for nodes to be allocated before returning.
+            timeout: Maximum time to wait in seconds. Only used when wait=True.
+
+        Returns:
+            A tuple of (nodes, node_ips) where nodes is a list of node names
+            and node_ips is a list of corresponding IP addresses.
+        """
+        # Wait for nodes to be allocated if requested
+        if wait:
+            if timeout is None:
+                timeout = _SLURM_DEFAULT_PROVISION_TIMEOUT
+            self.wait_for_job_nodes(job_id, timeout=timeout)
+
+        cmd = (
+            f'squeue -h --jobs {job_id} -o "%N" | tr \',\' \'\\n\' | '
+            f'while read node; do '
+            # TODO(kevin): Use json output for more robust parsing.
+            f'ip=$(scontrol show node=$node | grep NodeAddr= | '
+            f'awk -F= \'{{print $2}}\' | awk \'{{print $1}}\'); '
+            f'echo "$node $ip"; '
+            f'done')
+        rc, stdout, stderr = self._runner.run(cmd,
+                                              require_outputs=True,
+                                              stream_logs=False)
+        subprocess_utils.handle_returncode(
+            rc, cmd, f'Failed to get nodes for job {job_id}.', stderr=stderr)
+        logger.debug(f'Successfully got nodes for job {job_id}: {stdout}')
+
+        node_info = {}
+        for line in stdout.strip().splitlines():
+            line = line.strip()
+            if line:
+                parts = line.split()
+                if len(parts) >= 2:
+                    node_name = parts[0]
+                    node_ip = parts[1]
+                    node_info[node_name] = node_ip
+
+        nodes = list(node_info.keys())
+        node_ips = [node_info[node] for node in nodes]
+        if not nodes:
+            raise RuntimeError(
+                f'No nodes found for job {job_id}. '
+                f'The job may have terminated or the output was empty.')
+        assert (len(nodes) == len(node_ips)
+               ), f'Number of nodes and IPs do not match: {nodes} != {node_ips}'
+
+        return nodes, node_ips
+
+    def submit_job(
+        self,
+        partition: str,
+        job_name: str,
+        script_path: str,
+    ) -> str:
+        """Submit a Slurm job script.
+
+        Args:
+            partition: Slurm partition to submit to.
+            job_name: Name to give the job.
+            script_path: Remote path where the script will be stored.
+
+        Returns:
+            The job ID of the submitted job.
+        """
+        cmd = f'sbatch --partition={partition} {script_path}'
+        rc, stdout, stderr = self._runner.run(cmd,
+                                              require_outputs=True,
+                                              stream_logs=False)
+        subprocess_utils.handle_returncode(rc,
+                                           cmd,
+                                           'Failed to submit Slurm job.',
+                                           stderr=f'{stdout}\n{stderr}')
+
+        # Parse job ID from sbatch output (format: "Submitted batch job 12345")
+        job_id_match = re.search(r'Submitted batch job (\d+)', stdout)
+        if not job_id_match:
+            raise RuntimeError(
+                f'Failed to parse job ID from sbatch output: {stdout}')
+
+        job_id = job_id_match.group(1).strip()
+        logger.debug(f'Successfully submitted Slurm job {job_id} with name '
+                     f'{job_name}: {stdout}')
+
+        return job_id
+
+    def get_partitions_info(self) -> List[SlurmPartition]:
+        """Get the partitions information for the Slurm cluster.
+
+        Returns:
+            List of SlurmPartition objects.
+        """
+        cmd = 'scontrol show partitions -o'
+        rc, stdout, stderr = self._runner.run(cmd,
+                                              require_outputs=True,
+                                              stream_logs=False)
+        subprocess_utils.handle_returncode(rc,
+                                           cmd,
+                                           'Failed to get Slurm partitions.',
+                                           stderr=stderr)
+
+        partitions = []
+        for line in stdout.strip().splitlines():
+            is_default = False
+            match = _PARTITION_NAME_REGEX.search(line)
+            if 'Default=YES' in line:
+                is_default = True
+            if match:
+                partition = match.group(1).strip()
+                if partition:
+                    partitions.append(
+                        SlurmPartition(name=partition, is_default=is_default))
+        return partitions
+
+    def get_default_partition(self) -> Optional[str]:
+        """Get the default partition name for the Slurm cluster.
+
+        Returns:
+            The default partition name, or None if it cannot be determined.
+        """
+        partitions = self.get_partitions_info()
+        for partition in partitions:
+            if partition.is_default:
+                return partition.name
+        return None
+
+    def get_partitions(self) -> List[str]:
+        """Get unique partition names in the Slurm cluster.
+
+        Returns:
+            List of partition names. The default partition will not have a '*'
+            at the end of the name.
+        """
+        return [partition.name for partition in self.get_partitions_info()]

sky/backends/backend_utils.py

@@ -147,6 +147,19 @@ CLUSTER_TUNNEL_LOCK_TIMEOUT_SECONDS = 10.0
 # Remote dir that holds our runtime files.
 _REMOTE_RUNTIME_FILES_DIR = '~/.sky/.runtime_files'
 
+# The maximum size of a command line arguments is 128 KB, i.e. the command
+# executed with /bin/sh should be less than 128KB.
+# https://github.com/torvalds/linux/blob/master/include/uapi/linux/binfmts.h
+#
+# If a user have very long run or setup commands, the generated command may
+# exceed the limit, as we directly include scripts in job submission commands.
+# If the command is too long, we instead write it to a file, rsync and execute
+# it.
+#
+# We use 100KB as a threshold to be safe for other arguments that
+# might be added during ssh.
+_MAX_INLINE_SCRIPT_LENGTH = 100 * 1024
+
 _ENDPOINTS_RETRY_MESSAGE = ('If the cluster was recently started, '
                             'please retry after a while.')
 
@@ -225,6 +238,18 @@ _ACK_MESSAGE = 'ack'
 _FORWARDING_FROM_MESSAGE = 'Forwarding from'
 
 
+def is_command_length_over_limit(command: str) -> bool:
+    """Check if the length of the command exceeds the limit.
+
+    We calculate the length of the command after quoting the command twice as
+    when it is executed by the CommandRunner, the command will be quoted twice
+    to ensure the correctness, which will add significant length to the command.
+    """
+
+    quoted_length = len(shlex.quote(shlex.quote(command)))
+    return quoted_length > _MAX_INLINE_SCRIPT_LENGTH
+
+
 def is_ip(s: str) -> bool:
     """Returns whether this string matches IP_ADDR_REGEX."""
     return len(re.findall(IP_ADDR_REGEX, s)) == 1
@@ -946,6 +971,9 @@ def write_cluster_config(
                         '{conda_auto_activate}',
                         conda_auto_activate).replace('{is_custom_docker}',
                                                      is_custom_docker),
+                # Currently only used by Slurm. For other clouds, it is
+                # already part of ray_skypilot_installation_commands
+                'setup_sky_dirs_commands': constants.SETUP_SKY_DIRS_COMMANDS,
                 'ray_skypilot_installation_commands':
                     (constants.RAY_SKYPILOT_INSTALLATION_COMMANDS.replace(
                         '{sky_wheel_hash}',
@@ -1058,7 +1086,11 @@ def write_cluster_config(
         with open(tmp_yaml_path, 'w', encoding='utf-8') as f:
             f.write(restored_yaml_content)
 
-    config_dict['cluster_name_on_cloud'] = cluster_name_on_cloud
+    # Read the cluster_name_on_cloud from the restored yaml. This is a hack to
+    # make sure that launching on the same cluster across multiple users works
+    # correctly. See #8232.
+    yaml_config = yaml_utils.read_yaml(tmp_yaml_path)
+    config_dict['cluster_name_on_cloud'] = yaml_config['cluster_name']
 
     # Make sure to do this before we optimize file mounts. Optimization is
     # non-deterministic, but everything else before this point should be
@@ -1105,17 +1137,21 @@ def _add_auth_to_cluster_config(cloud: clouds.Cloud, tmp_yaml_path: str):
     """
     config = yaml_utils.read_yaml(tmp_yaml_path)
     # Check the availability of the cloud type.
-    if isinstance(cloud, (
+    if isinstance(
+            cloud,
+        (
             clouds.AWS,
             clouds.OCI,
             clouds.SCP,
+            # TODO(jwj): Handle Slurm-specific auth logic
+            clouds.Slurm,
             clouds.Vsphere,
             clouds.Cudo,
             clouds.Paperspace,
             clouds.Azure,
             clouds.DO,
             clouds.Nebius,
-    )):
+        )):
         config = auth.configure_ssh_info(config)
     elif isinstance(cloud, clouds.GCP):
         config = auth.setup_gcp_authentication(config)
@@ -2361,7 +2397,12 @@ def _update_cluster_status(
     # remain healthy for a while before the cloud completely preempts the VMs.
     # We have mitigated this by again first querying the VM state from the cloud
     # provider.
-    if all_nodes_up and run_ray_status_to_check_ray_cluster_healthy():
+    cloud = handle.launched_resources.cloud
+
+    # For Slurm, skip Ray health check since it doesn't use Ray.
+    should_check_ray = cloud is not None and cloud.uses_ray()
+    if all_nodes_up and (not should_check_ray or
+                         run_ray_status_to_check_ray_cluster_healthy()):
         # NOTE: all_nodes_up calculation is fast due to calling cloud CLI;
         # run_ray_status_to_check_all_nodes_up() is slow due to calling `ray get
         # head-ip/worker-ips`.