skypilot-nightly 1.0.0.dev20250311__tar.gz → 1.0.0.dev20250312__tar.gz

{skypilot_nightly-1.0.0.dev20250311/skypilot_nightly.egg-info → skypilot_nightly-1.0.0.dev20250312}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: skypilot-nightly
-Version: 1.0.0.dev20250311
+Version: 1.0.0.dev20250312
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0

{skypilot_nightly-1.0.0.dev20250311 → skypilot_nightly-1.0.0.dev20250312}/sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '439de1a24a6f0a9601051ecdc3e565308bac442a'
+_SKYPILOT_COMMIT_SHA = '78a42b6e733bbc29b68efe0e9c79191eaaca9fcd'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250311'
+__version__ = '1.0.0.dev20250312'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

{skypilot_nightly-1.0.0.dev20250311 → skypilot_nightly-1.0.0.dev20250312}/sky/adaptors/nebius.py

@@ -6,9 +6,11 @@ from sky.adaptors import common
 NEBIUS_TENANT_ID_FILENAME = 'NEBIUS_TENANT_ID.txt'
 NEBIUS_IAM_TOKEN_FILENAME = 'NEBIUS_IAM_TOKEN.txt'
 NEBIUS_PROJECT_ID_FILENAME = 'NEBIUS_PROJECT_ID.txt'
+NEBIUS_CREDENTIALS_FILENAME = 'credentials.json'
 NEBIUS_TENANT_ID_PATH = '~/.nebius/' + NEBIUS_TENANT_ID_FILENAME
 NEBIUS_IAM_TOKEN_PATH = '~/.nebius/' + NEBIUS_IAM_TOKEN_FILENAME
 NEBIUS_PROJECT_ID_PATH = '~/.nebius/' + NEBIUS_PROJECT_ID_FILENAME
+NEBIUS_CREDENTIALS_PATH = '~/.nebius/' + NEBIUS_CREDENTIALS_FILENAME
 
 MAX_RETRIES_TO_DISK_CREATE = 120
 MAX_RETRIES_TO_INSTANCE_STOP = 120
@@ -72,6 +74,11 @@ def get_iam_token():
     return _iam_token
 
 
+def is_token_or_cred_file_exist():
+    return (os.path.exists(os.path.expanduser(NEBIUS_IAM_TOKEN_PATH)) or
+            os.path.exists(os.path.expanduser(NEBIUS_CREDENTIALS_PATH)))
+
+
 def get_project_id():
     global _project_id
     if _project_id is None:
@@ -97,4 +104,7 @@ def get_tenant_id():
 
 
 def sdk():
-    return nebius.sdk.SDK(credentials=get_iam_token())
+    if get_iam_token() is not None:
+        return nebius.sdk.SDK(credentials=get_iam_token())
+    return nebius.sdk.SDK(
+        credentials_file_name=os.path.expanduser(NEBIUS_CREDENTIALS_PATH))

{skypilot_nightly-1.0.0.dev20250311 → skypilot_nightly-1.0.0.dev20250312}/sky/backends/backend_utils.py

@@ -1802,6 +1802,21 @@ def _update_cluster_status(cluster_name: str) -> Optional[Dict[str, Any]]:
         status == status_lib.ClusterStatus.UP for status in node_statuses) and
                     len(node_statuses) == handle.launched_nodes)
 
+    def get_node_counts_from_ray_status(
+            runner: command_runner.CommandRunner) -> Tuple[int, int, str, str]:
+        rc, output, stderr = runner.run(
+            instance_setup.RAY_STATUS_WITH_SKY_RAY_PORT_COMMAND,
+            stream_logs=False,
+            require_outputs=True,
+            separate_stderr=True)
+        if rc:
+            raise RuntimeError(
+                f'Refreshing status ({cluster_name!r}): Failed to check '
+                f'ray cluster\'s healthiness with '
+                f'{instance_setup.RAY_STATUS_WITH_SKY_RAY_PORT_COMMAND}.\n'
+                f'-- stdout --\n{output}\n-- stderr --\n{stderr}')
+        return (*_count_healthy_nodes_from_ray(output), output, stderr)
+
     def run_ray_status_to_check_ray_cluster_healthy() -> bool:
         try:
             # NOTE: fetching the IPs is very slow as it calls into
@@ -1822,26 +1837,34 @@ def _update_cluster_status(cluster_name: str) -> Optional[Dict[str, Any]]:
                 raise exceptions.FetchClusterInfoError(
                     reason=exceptions.FetchClusterInfoError.Reason.HEAD)
             head_runner = runners[0]
-            rc, output, stderr = head_runner.run(
-                instance_setup.RAY_STATUS_WITH_SKY_RAY_PORT_COMMAND,
-                stream_logs=False,
-                require_outputs=True,
-                separate_stderr=True)
-            if rc:
-                raise RuntimeError(
-                    f'Refreshing status ({cluster_name!r}): Failed to check '
-                    f'ray cluster\'s healthiness with '
-                    f'{instance_setup.RAY_STATUS_WITH_SKY_RAY_PORT_COMMAND}.\n'
-                    f'-- stdout --\n{output}\n-- stderr --\n{stderr}')
 
-            ready_head, ready_workers = _count_healthy_nodes_from_ray(output)
             total_nodes = handle.launched_nodes * handle.num_ips_per_node
-            if ready_head + ready_workers == total_nodes:
-                return True
+
+            for i in range(5):
+                ready_head, ready_workers, output, stderr = (
+                    get_node_counts_from_ray_status(head_runner))
+                if ready_head + ready_workers == total_nodes:
+                    return True
+                logger.debug(f'Refreshing status ({cluster_name!r}) attempt '
+                             f'{i}: ray status not showing all nodes '
+                             f'({ready_head + ready_workers}/{total_nodes});\n'
+                             f'output:\n{output}\nstderr:\n{stderr}')
+
+                # If cluster JUST started, maybe not all the nodes have shown
+                # up. Try again for a few seconds.
+                # Note: We are okay with this performance hit because it's very
+                # rare to normally hit this case. It requires:
+                # - All the instances in the cluster are up on the cloud side
+                #   (not preempted), but
+                # - The ray cluster is somehow degraded so not all instances are
+                #   showing up
+                time.sleep(1)
+
             raise RuntimeError(
                 f'Refreshing status ({cluster_name!r}): ray status not showing '
                 f'all nodes ({ready_head + ready_workers}/'
-                f'{total_nodes}); output: {output}; stderr: {stderr}')
+                f'{total_nodes});\noutput:\n{output}\nstderr:\n{stderr}')
+
         except exceptions.FetchClusterInfoError:
             logger.debug(
                 f'Refreshing status ({cluster_name!r}) failed to get IPs.')

{skypilot_nightly-1.0.0.dev20250311 → skypilot_nightly-1.0.0.dev20250312}/sky/backends/cloud_vm_ray_backend.py

@@ -772,32 +772,6 @@ class FailoverCloudErrorHandlerV1:
             setattr(e, 'detailed_reason', detailed_reason)
             raise e
 
-    @staticmethod
-    def _lambda_handler(blocked_resources: Set['resources_lib.Resources'],
-                        launchable_resources: 'resources_lib.Resources',
-                        region: 'clouds.Region',
-                        zones: Optional[List['clouds.Zone']], stdout: str,
-                        stderr: str):
-        del region, zones  # Unused.
-        errors = FailoverCloudErrorHandlerV1._handle_errors(
-            stdout,
-            stderr,
-            is_error_str_known=lambda x: 'LambdaCloudError:' in x.strip())
-        messages = '\n  '.join(errors)
-        style = colorama.Style
-        logger.warning(f'  {style.DIM}{messages}{style.RESET_ALL}')
-        _add_to_blocked_resources(blocked_resources,
-                                  launchable_resources.copy(zone=None))
-
-        # Sometimes, LambdaCloudError will list available regions.
-        for e in errors:
-            if e.find('Regions with capacity available:') != -1:
-                for r in service_catalog.regions('lambda'):
-                    if e.find(r.name) == -1:
-                        _add_to_blocked_resources(
-                            blocked_resources,
-                            launchable_resources.copy(region=r.name, zone=None))
-
     @staticmethod
     def _scp_handler(blocked_resources: Set['resources_lib.Resources'],
                      launchable_resources: 'resources_lib.Resources',
@@ -846,32 +820,6 @@ class FailoverCloudErrorHandlerV1:
             _add_to_blocked_resources(blocked_resources,
                                       launchable_resources.copy(zone=zone.name))
 
-    # Apr, 2023 by Hysun(hysun.he@oracle.com): Added support for OCI
-    @staticmethod
-    def _oci_handler(blocked_resources: Set['resources_lib.Resources'],
-                     launchable_resources: 'resources_lib.Resources',
-                     region: 'clouds.Region',
-                     zones: Optional[List['clouds.Zone']], stdout: str,
-                     stderr: str):
-        known_service_errors = [
-            'NotAuthorizedOrNotFound', 'CannotParseRequest', 'InternalError',
-            'LimitExceeded', 'NotAuthenticated'
-        ]
-        errors = FailoverCloudErrorHandlerV1._handle_errors(
-            stdout, stderr, lambda x: 'VcnSubnetNotFound' in x.strip() or
-            ('oci.exceptions.ServiceError' in x.strip() and any(
-                known_err in x.strip() for known_err in known_service_errors)))
-        logger.warning(f'Got error(s) in {region.name}:')
-        messages = '\n\t'.join(errors)
-        style = colorama.Style
-        logger.warning(f'{style.DIM}\t{messages}{style.RESET_ALL}')
-
-        if zones is not None:
-            for zone in zones:
-                _add_to_blocked_resources(
-                    blocked_resources,
-                    launchable_resources.copy(zone=zone.name))
-
     @staticmethod
     def update_blocklist_on_error(
             blocked_resources: Set['resources_lib.Resources'],
@@ -1123,6 +1071,23 @@ class FailoverCloudErrorHandlerV2:
                     blocked_resources,
                     launchable_resources.copy(zone=zone.name))
 
+    @staticmethod
+    def _lambda_handler(blocked_resources: Set['resources_lib.Resources'],
+                        launchable_resources: 'resources_lib.Resources',
+                        region: 'clouds.Region',
+                        zones: Optional[List['clouds.Zone']], error: Exception):
+        output = str(error)
+        # Sometimes, lambda cloud error will list available regions.
+        if output.find('Regions with capacity available:') != -1:
+            for r in service_catalog.regions('lambda'):
+                if output.find(r.name) == -1:
+                    _add_to_blocked_resources(
+                        blocked_resources,
+                        launchable_resources.copy(region=r.name, zone=None))
+        else:
+            FailoverCloudErrorHandlerV2._default_handler(
+                blocked_resources, launchable_resources, region, zones, error)
+
     @staticmethod
     def _default_handler(blocked_resources: Set['resources_lib.Resources'],
                          launchable_resources: 'resources_lib.Resources',

{skypilot_nightly-1.0.0.dev20250311 → skypilot_nightly-1.0.0.dev20250312}/sky/clouds/nebius.py

@@ -17,6 +17,7 @@ _CREDENTIAL_FILES = [
     nebius.NEBIUS_TENANT_ID_FILENAME,
     nebius.NEBIUS_IAM_TOKEN_FILENAME,
     nebius.NEBIUS_PROJECT_ID_FILENAME,
+    nebius.NEBIUS_CREDENTIALS_FILENAME
 ]
 
 
@@ -252,15 +253,16 @@ class Nebius(clouds.Cloud):
     def check_credentials(cls) -> Tuple[bool, Optional[str]]:
         """ Verify that the user has valid credentials for Nebius. """
         logging.debug('Nebius cloud check credentials')
-        token = nebius.get_iam_token()
-        token_msg = ('    Credentials can be set up by running: \n'\
-                    f'        $ nebius iam get-access-token > {nebius.NEBIUS_IAM_TOKEN_PATH} \n')  # pylint: disable=line-too-long
+        token_cred_msg = ('    Credentials can be set up by running: \n'\
+                    f'        $ nebius iam get-access-token > {nebius.NEBIUS_IAM_TOKEN_PATH} \n'\
+                          '    or generate  ~/.nebius/credentials.json')  # pylint: disable=line-too-long
+
         tenant_msg = ('   Copy your tenat ID from the web console and save it to file \n'  # pylint: disable=line-too-long
                       f'        $ echo $NEBIUS_TENANT_ID_PATH > {nebius.NEBIUS_TENANT_ID_PATH} \n'  # pylint: disable=line-too-long
                       '   Or if you have 1 tenant you can run:\n'  # pylint: disable=line-too-long
                       f'        $ nebius --format json iam whoami|jq -r \'.user_profile.tenants[0].tenant_id\' > {nebius.NEBIUS_TENANT_ID_PATH} \n')  # pylint: disable=line-too-long
-        if token is None:
-            return False, f'{token_msg}'
+        if not nebius.is_token_or_cred_file_exist():
+            return False, f'{token_cred_msg}'
         sdk = nebius.sdk()
         tenant_id = nebius.get_tenant_id()
         if tenant_id is None:
@@ -272,7 +274,7 @@ class Nebius(clouds.Cloud):
         except nebius.request_error() as e:
             return False, (
                 f'{e.status} \n'  # First line is indented by 4 spaces
-                f'{token_msg}'
+                f'{token_cred_msg}'
                 f'{tenant_msg}')
         return True, None
 

{skypilot_nightly-1.0.0.dev20250311 → skypilot_nightly-1.0.0.dev20250312}/sky/exceptions.py

@@ -28,12 +28,19 @@ GIT_FATAL_EXIT_CODE = 128
 ARCH_NOT_SUPPORTED_EXIT_CODE = 133
 
 
-def is_safe_exception(exc: Exception) -> bool:
+def is_safe_exception(exc: BaseException) -> bool:
     """Returns True if the exception is safe to send to clients.
 
     Safe exceptions are:
     1. Built-in exceptions
     2. SkyPilot's own exceptions
+
+    Args:
+        exc: The exception to check, accept BaseException to handle SystemExit
+            and KeyboardInterrupt.
+
+    Returns:
+        True if the exception is safe to send to clients, False otherwise.
     """
     module = type(exc).__module__
 
@@ -48,7 +55,7 @@ def is_safe_exception(exc: Exception) -> bool:
     return False
 
 
-def wrap_exception(exc: Exception) -> Exception:
+def wrap_exception(exc: BaseException) -> BaseException:
     """Wraps non-safe exceptions into SkyPilot exceptions
 
     This is used to wrap exceptions that are not safe to deserialize at clients.
@@ -64,7 +71,8 @@ def wrap_exception(exc: Exception) -> Exception:
                       error_type=type(exc).__name__)
 
 
-def serialize_exception(e: Exception) -> Dict[str, Any]:
+# Accept BaseException to handle SystemExit and KeyboardInterrupt
+def serialize_exception(e: BaseException) -> Dict[str, Any]:
     """Serialize the exception.
 
     This function also wraps any unsafe exceptions (e.g., cloud exceptions)

{skypilot_nightly-1.0.0.dev20250311 → skypilot_nightly-1.0.0.dev20250312}/sky/provision/kubernetes/utils.py

@@ -853,7 +853,7 @@ def get_accelerator_label_key_value(
                 for label, value in label_list:
                     if (label_formatter.match_label_key(label) and
                             label_formatter.get_accelerator_from_label_value(
-                                value) == acc_type):
+                                value).lower() == acc_type.lower()):
                         if is_tpu_on_gke(acc_type):
                             assert isinstance(label_formatter,
                                               GKELabelFormatter)

skypilot_nightly-1.0.0.dev20250312/sky/server/requests/event_loop.py

@@ -0,0 +1,31 @@
+"""Executor event loop to process tasks in coroutines."""
+import asyncio
+import concurrent.futures
+import threading
+from typing import Coroutine, Optional
+
+# Dedicated event loop for requests, isolated with the event loop managed
+# by uvicorn. This is responsible for light-weight async tasks or sub-tasks,
+# refer to `executor.py` for more details about cooperation between the event
+# loop and executor process pool.
+_EVENT_LOOP: Optional[asyncio.AbstractEventLoop] = None
+_LOCK = threading.Lock()
+
+
+def run(coro: Coroutine) -> concurrent.futures.Future:
+    """Run a coroutine asynchronously in the request event loop."""
+    return asyncio.run_coroutine_threadsafe(coro, get_event_loop())
+
+
+def get_event_loop() -> asyncio.AbstractEventLoop:
+    """Open and get the event loop."""
+    global _EVENT_LOOP
+    if _EVENT_LOOP is not None and not _EVENT_LOOP.is_closed():
+        return _EVENT_LOOP
+    with _LOCK:
+        if _EVENT_LOOP is None or _EVENT_LOOP.is_closed():
+            _EVENT_LOOP = asyncio.new_event_loop()
+            loop_thread = threading.Thread(target=_EVENT_LOOP.run_forever,
+                                           daemon=True)
+            loop_thread.start()
+    return _EVENT_LOOP

{skypilot_nightly-1.0.0.dev20250311 → skypilot_nightly-1.0.0.dev20250312}/sky/server/requests/executor.py

@@ -27,8 +27,8 @@ import os
 import queue as queue_lib
 import signal
 import sys
+import threading
 import time
-import traceback
 import typing
 from typing import Any, Callable, Generator, List, Optional, TextIO, Tuple
 
@@ -41,11 +41,13 @@ from sky import skypilot_config
 from sky.server import common as server_common
 from sky.server import constants as server_constants
 from sky.server.requests import payloads
+from sky.server.requests import preconditions
 from sky.server.requests import requests as api_requests
 from sky.server.requests.queues import mp_queue
 from sky.skylet import constants
 from sky.utils import annotations
 from sky.utils import common_utils
+from sky.utils import subprocess_utils
 from sky.utils import timeline
 from sky.utils import ux_utils
 
@@ -262,13 +264,7 @@ def _request_execution_wrapper(request_id: str,
             _restore_output(original_stdout, original_stderr)
             return
         except (Exception, SystemExit) as e:  # pylint: disable=broad-except
-            with ux_utils.enable_traceback():
-                stacktrace = traceback.format_exc()
-            setattr(e, 'stacktrace', stacktrace)
-            with api_requests.update_request(request_id) as request_task:
-                assert request_task is not None, request_id
-                request_task.status = api_requests.RequestStatus.FAILED
-                request_task.set_error(e)
+            api_requests.set_request_failed(request_id, e)
             _restore_output(original_stdout, original_stderr)
             logger.info(f'Request {request_id} failed due to '
                         f'{common_utils.format_exception(e)}')
@@ -283,16 +279,37 @@ def _request_execution_wrapper(request_id: str,
             logger.info(f'Request {request_id} finished')
 
 
-def schedule_request(request_id: str,
-                     request_name: str,
-                     request_body: payloads.RequestBody,
-                     func: Callable[P, Any],
-                     request_cluster_name: Optional[str] = None,
-                     ignore_return_value: bool = False,
-                     schedule_type: api_requests.ScheduleType = api_requests.
-                     ScheduleType.LONG,
-                     is_skypilot_system: bool = False) -> None:
-    """Enqueue a request to the request queue."""
+def schedule_request(
+        request_id: str,
+        request_name: str,
+        request_body: payloads.RequestBody,
+        func: Callable[P, Any],
+        request_cluster_name: Optional[str] = None,
+        ignore_return_value: bool = False,
+        schedule_type: api_requests.ScheduleType = (
+            api_requests.ScheduleType.LONG),
+        is_skypilot_system: bool = False,
+        precondition: Optional[preconditions.Precondition] = None) -> None:
+    """Enqueue a request to the request queue.
+
+    Args:
+        request_id: ID of the request.
+        request_name: Name of the request type, e.g. "sky.launch".
+        request_body: The request body containing parameters and environment
+            variables.
+        func: The function to execute when the request is processed.
+        request_cluster_name: The name of the cluster associated with this
+            request, if any.
+        ignore_return_value: If True, the return value of the function will be
+            ignored.
+        schedule_type: The type of scheduling to use for this request, refer to
+            `api_requests.ScheduleType` for more details.
+        is_skypilot_system: Denote whether the request is from SkyPilot system.
+        precondition: If a precondition is provided, the request will only be
+            scheduled for execution when the precondition is met (returns True).
+            The precondition is waited asynchronously and does not block the
+            caller.
+    """
     user_id = request_body.env_vars[constants.USER_ID_ENV_VAR]
     if is_skypilot_system:
         user_id = server_constants.SKYPILOT_SYSTEM_USER_ID
@@ -314,10 +331,17 @@ def schedule_request(request_id: str,
         return
 
     request.log_path.touch()
-    input_tuple = (request_id, ignore_return_value)
 
-    logger.info(f'Queuing request: {request_id}')
-    _get_queue(schedule_type).put(input_tuple)
+    def enqueue():
+        input_tuple = (request_id, ignore_return_value)
+        logger.info(f'Queuing request: {request_id}')
+        _get_queue(schedule_type).put(input_tuple)
+
+    if precondition is not None:
+        # Wait async to avoid blocking caller.
+        precondition.wait_async(on_condition_met=enqueue)
+    else:
+        enqueue()
 
 
 def executor_initializer(proc_group: str):
@@ -431,13 +455,17 @@ def start(deploy: bool) -> List[multiprocessing.Process]:
 
     logger.info('Request queues created')
 
+    long_workers = []
     for worker_id in range(max_parallel_for_long):
         worker = RequestWorker(id=worker_id,
                                schedule_type=api_requests.ScheduleType.LONG)
         worker_proc = multiprocessing.Process(target=request_worker,
                                               args=(worker, 1))
-        worker_proc.start()
+        long_workers.append(worker_proc)
         sub_procs.append(worker_proc)
+    threading.Thread(target=subprocess_utils.slow_start_processes,
+                     args=(long_workers,),
+                     daemon=True).start()
 
     # Start a worker for short requests.
     worker = RequestWorker(id=1, schedule_type=api_requests.ScheduleType.SHORT)

skypilot_nightly-1.0.0.dev20250312/sky/server/requests/preconditions.py

@@ -0,0 +1,174 @@
+"""Precondition for a request to be executed.
+
+Preconditions are introduced so that:
+- Wait for precondition does not block executor process, which is expensive;
+- Cross requests knowledge (e.g. waiting for other requests to be completed)
+  can be handled at precondition level, instead of invading the execution
+  logic of specific requests.
+"""
+import abc
+import asyncio
+import time
+from typing import Callable, Optional, Tuple
+
+from sky import exceptions
+from sky import global_user_state
+from sky import sky_logging
+from sky.server.requests import event_loop
+from sky.server.requests import requests as api_requests
+from sky.utils import common_utils
+from sky.utils import status_lib
+
+# The default interval seconds to check the precondition.
+_PRECONDITION_CHECK_INTERVAL = 1
+# The default timeout seconds to wait for the precondition to be met.
+_PRECONDITION_TIMEOUT = 60 * 60
+
+logger = sky_logging.init_logger(__name__)
+
+
+class Precondition(abc.ABC):
+    """Abstract base class for a precondition for a request to be executed.
+
+    A Precondition can be waited in either of the following ways:
+    - await Precondition: wait for the precondition to be met.
+    - Precondition.wait_async: wait for the precondition to be met in background
+      and execute the given callback on met.
+    """
+
+    def __init__(self,
+                 request_id: str,
+                 check_interval: float = _PRECONDITION_CHECK_INTERVAL,
+                 timeout: float = _PRECONDITION_TIMEOUT):
+        self.request_id = request_id
+        self.check_interval = check_interval
+        self.timeout = timeout
+
+    def __await__(self):
+        """Make Precondition awaitable."""
+        return self._wait().__await__()
+
+    def wait_async(
+            self,
+            on_condition_met: Optional[Callable[[], None]] = None) -> None:
+        """Wait precondition asynchronously and execute the callback on met."""
+
+        async def wait_with_callback():
+            met = await self
+            if met and on_condition_met is not None:
+                on_condition_met()
+
+        event_loop.run(wait_with_callback())
+
+    @abc.abstractmethod
+    async def check(self) -> Tuple[bool, Optional[str]]:
+        """Check if the precondition is met.
+
+        Note that compared to _request_execution_wrapper, the env vars and
+        skypilot config here are not overridden since the lack of process
+        isolation, which may cause issues if the check accidentally depends on
+        these. Make sure the check function is independent of the request
+        environment.
+        TODO(aylei): a new request context isolation mechanism is needed to
+        enable more tasks/sub-tasks to be processed in coroutines or threads.
+
+        Returns:
+            A tuple of (bool, Optional[str]).
+            The bool indicates if the precondition is met.
+            The str is the current status of the precondition if any.
+        """
+        raise NotImplementedError
+
+    async def _wait(self) -> bool:
+        """Wait for the precondition to be met.
+
+        Args:
+            on_condition_met: Callback to execute when the precondition is met.
+        """
+        start_time = time.time()
+        last_status_msg = ''
+        while True:
+            if self.timeout > 0 and time.time() - start_time > self.timeout:
+                # Cancel the request on timeout.
+                api_requests.set_request_failed(
+                    self.request_id,
+                    exceptions.RequestCancelled(
+                        f'Request {self.request_id} precondition wait timed '
+                        f'out after {self.timeout}s'))
+                return False
+
+            # Check if the request has been cancelled
+            request = api_requests.get_request(self.request_id)
+            if request is None:
+                logger.error(f'Request {self.request_id} not found')
+                return False
+            if request.status == api_requests.RequestStatus.CANCELLED:
+                logger.debug(f'Request {self.request_id} cancelled')
+                return False
+
+            try:
+                met, status_msg = await self.check()
+                if met:
+                    return True
+                if status_msg is not None and status_msg != last_status_msg:
+                    # Update the status message if it has changed.
+                    with api_requests.update_request(self.request_id) as req:
+                        assert req is not None, self.request_id
+                        req.status_msg = status_msg
+                    last_status_msg = status_msg
+            except (Exception, SystemExit, KeyboardInterrupt) as e:  # pylint: disable=broad-except
+                api_requests.set_request_failed(self.request_id, e)
+                logger.info(f'Request {self.request_id} failed due to '
+                            f'{common_utils.format_exception(e)}')
+                return False
+
+            await asyncio.sleep(self.check_interval)
+
+
+class ClusterStartCompletePrecondition(Precondition):
+    """Whether the start process of a cluster is complete.
+
+    This condition only waits the start process of a cluster to complete, e.g.
+    `sky launch` or `sky start`.
+    For cluster that has been started but not in UP status, bypass the waiting
+    in favor of:
+    - allowing the task to refresh cluster status from cloud vendor;
+    - unified error message in task handlers.
+
+    Args:
+        request_id: The request ID of the task.
+        cluster_name: The name of the cluster to wait for.
+    """
+
+    def __init__(self, request_id: str, cluster_name: str, **kwargs):
+        super().__init__(request_id=request_id, **kwargs)
+        self.cluster_name = cluster_name
+
+    async def check(self) -> Tuple[bool, Optional[str]]:
+        cluster_record = global_user_state.get_cluster_from_name(
+            self.cluster_name)
+        if (cluster_record and
+                cluster_record['status'] is status_lib.ClusterStatus.UP):
+            # Shortcut for started clusters, ignore cluster not found
+            # since the cluster record might not yet be created by the
+            # launch task.
+            return True, None
+        # Check if there is a task starting the cluster, we do not check
+        # SUCCEEDED requests since successfully launched cluster can be
+        # restarted later on.
+        # Note that since the requests are not persistent yet between restarts,
+        # a cluster might be started in halfway and requests are lost.
+        # We unify these situations into a single state: the process of starting
+        # the cluster is done (either normally or abnormally) but cluster is not
+        # in UP status.
+        requests = api_requests.get_request_tasks(
+            status=[
+                api_requests.RequestStatus.RUNNING,
+                api_requests.RequestStatus.PENDING
+            ],
+            include_request_names=['sky.launch', 'sky.start'],
+            cluster_names=[self.cluster_name])
+        if len(requests) == 0:
+            # No runnning or pending tasks, the start process is done.
+            return True, None
+        return False, f'Waiting for cluster {self.cluster_name} to be UP.'