skypilot-nightly 1.0.0.dev20250922__py3-none-any.whl → 1.0.0.dev20250926__py3-none-any.whl

sky/serve/serve_utils.py

@@ -262,7 +262,7 @@ def _validate_consolidation_mode_config(current_is_consolidation_mode: bool,
     controller = controller_utils.get_controller_for_pool(pool).value
     if current_is_consolidation_mode:
         controller_cn = controller.cluster_name
-        if global_user_state.get_cluster_from_name(controller_cn) is not None:
+        if global_user_state.cluster_with_name_exists(controller_cn):
             with ux_utils.print_exception_no_traceback():
                 raise exceptions.InconsistentConsolidationModeError(
                     f'{colorama.Fore.RED}Consolidation mode for '
@@ -896,8 +896,8 @@ def _terminate_failed_services(
     # replicas, so we don't need to try again here.
     for replica_info in serve_state.get_replica_infos(service_name):
         # TODO(tian): Refresh latest status of the cluster.
-        if global_user_state.get_cluster_from_name(
-                replica_info.cluster_name) is not None:
+        if global_user_state.cluster_with_name_exists(
+                replica_info.cluster_name):
             remaining_replica_clusters.append(f'{replica_info.cluster_name!r}')
         serve_state.remove_replica(service_name, replica_info.replica_id)
 
@@ -1133,10 +1133,8 @@ def _process_line(line: str,
     # `✓ Cluster launched: new-http.  View logs at: *.log`
     # We should tail the detailed logs for user.
     def cluster_is_up() -> bool:
-        cluster_record = global_user_state.get_cluster_from_name(cluster_name)
-        if cluster_record is None:
-            return False
-        return cluster_record['status'] == status_lib.ClusterStatus.UP
+        status = global_user_state.get_status_from_cluster_name(cluster_name)
+        return status == status_lib.ClusterStatus.UP
 
     provision_api_log_prompt = re.match(_SKYPILOT_PROVISION_API_LOG_PATTERN,
                                         line)

sky/serve/server/core.py

@@ -46,20 +46,23 @@ def up(
 
 
 @usage_lib.entrypoint
-def update(
-        task: 'sky.Task',
-        service_name: str,
-        mode: serve_utils.UpdateMode = serve_utils.DEFAULT_UPDATE_MODE) -> None:
+def update(task: Optional['sky.Task'],
+           service_name: str,
+           mode: serve_utils.UpdateMode = serve_utils.DEFAULT_UPDATE_MODE,
+           workers: Optional[int] = None) -> None:
     """Updates an existing service.
 
     Please refer to the sky.cli.serve_update for the document.
 
     Args:
-        task: sky.Task to update.
+        task: sky.Task to update, or None if updating
+            the number of workers/replicas.
         service_name: Name of the service.
         mode: Update mode.
+        workers: Number of workers/replicas to set for the service when
+            task is None.
     """
-    return impl.update(task, service_name, mode, pool=False)
+    return impl.update(task, service_name, mode, pool=False, workers=workers)
 
 
 @usage_lib.entrypoint

sky/serve/server/impl.py

@@ -411,6 +411,9 @@ def up(
                 f'\n{ux_utils.INDENT_LAST_SYMBOL}To terminate the pool:\t'
                 f'{ux_utils.BOLD}sky jobs pool down {service_name}'
                 f'{ux_utils.RESET_BOLD}'
+                f'\n{ux_utils.INDENT_SYMBOL}To update the number of workers:\t'
+                f'{ux_utils.BOLD}sky jobs pool apply --pool {service_name} '
+                f'--workers 5{ux_utils.RESET_BOLD}'
                 '\n\n' + ux_utils.finishing_message('Successfully created pool '
                                                     f'{service_name!r}.'))
         else:
@@ -448,37 +451,15 @@ def up(
 
 
 def update(
-    task: 'task_lib.Task',
+    task: Optional['task_lib.Task'],
     service_name: str,
     mode: serve_utils.UpdateMode = serve_utils.DEFAULT_UPDATE_MODE,
     pool: bool = False,
+    workers: Optional[int] = None,
 ) -> None:
     """Updates an existing service or pool."""
     noun = 'pool' if pool else 'service'
     capnoun = noun.capitalize()
-    task.validate()
-    serve_utils.validate_service_task(task, pool=pool)
-
-    # Always apply the policy again here, even though it might have been applied
-    # in the CLI. This is to ensure that we apply the policy to the final DAG
-    # and get the mutated config.
-    # TODO(cblmemo,zhwu): If a user sets a new skypilot_config, the update
-    # will not apply the config.
-    dag, _ = admin_policy_utils.apply(task)
-    task = dag.tasks[0]
-    if pool:
-        if task.run is not None:
-            logger.warning(f'{colorama.Fore.YELLOW}The `run` section will be '
-                           f'ignored for pool.{colorama.Style.RESET_ALL}')
-        # Use dummy run script for cluster pool.
-        task.run = serve_constants.POOL_DUMMY_RUN_COMMAND
-
-    assert task.service is not None
-    if not pool and task.service.tls_credential is not None:
-        logger.warning('Updating TLS keyfile and certfile is not supported. '
-                       'Any updates to the keyfile and certfile will not take '
-                       'effect. To update TLS keyfile and certfile, please '
-                       'tear down the service and spin up a new one.')
 
     controller_type = controller_utils.get_controller_for_pool(pool)
     handle = backend_utils.is_controller_accessible(
@@ -505,6 +486,77 @@ def update(
                                f'To spin up a {noun}, use {ux_utils.BOLD}'
                                f'{cmd}{ux_utils.RESET_BOLD}')
 
+    # If task is None and workers is specified, load existing configuration
+    # and update replica count.
+    if task is None:
+        if workers is None:
+            with ux_utils.print_exception_no_traceback():
+                raise ValueError(
+                    f'Cannot update {noun} without specifying '
+                    f'task or workers. Please provide either a task '
+                    f'or specify the number of workers.')
+
+        if not pool:
+            with ux_utils.print_exception_no_traceback():
+                raise ValueError(
+                    'Non-pool service, trying to update replicas to '
+                    f'{workers} is not supported. Ignoring the update.')
+
+        # Load the existing task configuration from the service's YAML file
+        latest_yaml_path = serve_utils.generate_task_yaml_file_name(
+            service_name, service_record['version'], expand_user=False)
+
+        logger.debug('Loading existing task configuration from '
+                     f'{latest_yaml_path} to create a new modified task.')
+
+        # Get the path locally.
+        with tempfile.NamedTemporaryFile(
+                prefix=f'service-task-{service_name}-',
+                mode='w',
+        ) as service_file:
+            try:
+                backend.download_file(handle, latest_yaml_path,
+                                      service_file.name)
+            except exceptions.CommandError as e:
+                raise RuntimeError(
+                    f'Failed to download the old task configuration from '
+                    f'{latest_yaml_path}: {e.error_msg}') from e
+
+            # Load the existing task configuration
+            existing_config = yaml_utils.read_yaml(service_file.name)
+            task = task_lib.Task.from_yaml_config(existing_config)
+
+        if task.service is None:
+            with ux_utils.print_exception_no_traceback():
+                raise RuntimeError('No service configuration found in '
+                                   f'existing {noun} {service_name!r}')
+        task.set_service(task.service.copy(min_replicas=workers))
+
+    task.validate()
+    serve_utils.validate_service_task(task, pool=pool)
+
+    # Now apply the policy and handle task-specific logic
+    # Always apply the policy again here, even though it might have been applied
+    # in the CLI. This is to ensure that we apply the policy to the final DAG
+    # and get the mutated config.
+    # TODO(cblmemo,zhwu): If a user sets a new skypilot_config, the update
+    # will not apply the config.
+    dag, _ = admin_policy_utils.apply(task)
+    task = dag.tasks[0]
+    if pool:
+        if task.run is not None:
+            logger.warning(f'{colorama.Fore.YELLOW}The `run` section will be '
+                           f'ignored for pool.{colorama.Style.RESET_ALL}')
+        # Use dummy run script for cluster pool.
+        task.run = serve_constants.POOL_DUMMY_RUN_COMMAND
+
+    assert task.service is not None
+    if not pool and task.service.tls_credential is not None:
+        logger.warning('Updating TLS keyfile and certfile is not supported. '
+                       'Any updates to the keyfile and certfile will not take '
+                       'effect. To update TLS keyfile and certfile, please '
+                       'tear down the service and spin up a new one.')
+
     prompt = None
     if (service_record['status'] == serve_state.ServiceStatus.CONTROLLER_FAILED
        ):
@@ -625,6 +677,7 @@ def update(
 
 def apply(
     task: 'task_lib.Task',
+    workers: Optional[int],
     service_name: str,
     mode: serve_utils.UpdateMode = serve_utils.DEFAULT_UPDATE_MODE,
     pool: bool = False,
@@ -640,7 +693,7 @@ def apply(
             service_record = _get_service_record(service_name, pool, handle,
                                                  backend)
             if service_record is not None:
-                return update(task, service_name, mode, pool)
+                return update(task, service_name, mode, pool, workers)
         except exceptions.ClusterNotUpError:
             pass
         up(task, service_name, pool)

sky/serve/server/server.py

@@ -98,7 +98,7 @@ async def tail_logs(
     request: fastapi.Request, log_body: payloads.ServeLogsBody,
     background_tasks: fastapi.BackgroundTasks
 ) -> fastapi.responses.StreamingResponse:
-    executor.schedule_request(
+    request_task = executor.prepare_request(
         request_id=request.state.request_id,
         request_name='serve.logs',
         request_body=log_body,
@@ -106,10 +106,9 @@ async def tail_logs(
         schedule_type=api_requests.ScheduleType.SHORT,
         request_cluster_name=common.SKY_SERVE_CONTROLLER_NAME,
     )
-
-    request_task = await api_requests.get_request_async(request.state.request_id
-                                                       )
-
+    task = executor.execute_request_in_coroutine(request_task)
+    # Cancel the coroutine after the request is done or client disconnects
+    background_tasks.add_task(task.cancel)
     return stream_utils.stream_response(
         request_id=request_task.request_id,
         logs_path=request_task.log_path,

sky/serve/service_spec.py

@@ -506,3 +506,36 @@ class SkyServiceSpec:
         if not hasattr(self, '_pool'):
             return False
         return bool(self._pool)
+
+    def copy(self, **override) -> 'SkyServiceSpec':
+        return SkyServiceSpec(
+            readiness_path=override.pop('readiness_path', self._readiness_path),
+            initial_delay_seconds=override.pop('initial_delay_seconds',
+                                               self._initial_delay_seconds),
+            readiness_timeout_seconds=override.pop(
+                'readiness_timeout_seconds', self._readiness_timeout_seconds),
+            min_replicas=override.pop('min_replicas', self._min_replicas),
+            max_replicas=override.pop('max_replicas', self._max_replicas),
+            num_overprovision=override.pop('num_overprovision',
+                                           self._num_overprovision),
+            ports=override.pop('ports', self._ports),
+            target_qps_per_replica=override.pop('target_qps_per_replica',
+                                                self._target_qps_per_replica),
+            post_data=override.pop('post_data', self._post_data),
+            tls_credential=override.pop('tls_credential', self._tls_credential),
+            readiness_headers=override.pop('readiness_headers',
+                                           self._readiness_headers),
+            dynamic_ondemand_fallback=override.pop(
+                'dynamic_ondemand_fallback', self._dynamic_ondemand_fallback),
+            base_ondemand_fallback_replicas=override.pop(
+                'base_ondemand_fallback_replicas',
+                self._base_ondemand_fallback_replicas),
+            spot_placer=override.pop('spot_placer', self._spot_placer),
+            upscale_delay_seconds=override.pop('upscale_delay_seconds',
+                                               self._upscale_delay_seconds),
+            downscale_delay_seconds=override.pop('downscale_delay_seconds',
+                                                 self._downscale_delay_seconds),
+            load_balancing_policy=override.pop('load_balancing_policy',
+                                               self._load_balancing_policy),
+            pool=override.pop('pool', self._pool),
+        )

sky/server/constants.py

@@ -10,7 +10,7 @@ from sky.skylet import constants
 # based on version info is needed.
 # For more details and code guidelines, refer to:
 # https://docs.skypilot.co/en/latest/developers/CONTRIBUTING.html#backward-compatibility-guidelines
-API_VERSION = 18
+API_VERSION = 20
 
 # The minimum peer API version that the code should still work with.
 # Notes (dev):

sky/server/daemons.py

@@ -8,7 +8,6 @@ from sky import sky_logging
 from sky import skypilot_config
 from sky.server import constants as server_constants
 from sky.utils import annotations
-from sky.utils import common
 from sky.utils import common_utils
 from sky.utils import env_options
 from sky.utils import subprocess_utils
@@ -94,13 +93,13 @@ class InternalRequestDaemon:
 def refresh_cluster_status_event():
     """Periodically refresh the cluster status."""
     # pylint: disable=import-outside-toplevel
-    from sky import core
+    from sky.backends import backend_utils
 
     logger.info('=== Refreshing cluster status ===')
     # This periodically refresh will hold the lock for the cluster being
     # refreshed, but it is OK because other operations will just wait for
     # the lock and get the just refreshed status without refreshing again.
-    core.status(refresh=common.StatusRefreshMode.FORCE, all_users=True)
+    backend_utils.refresh_cluster_records()
     logger.info('Status refreshed. Sleeping '
                 f'{server_constants.CLUSTER_REFRESH_DAEMON_INTERVAL_SECONDS}'
                 ' seconds for the next refresh...\n')

sky/server/requests/executor.py

@@ -502,7 +502,35 @@ def _record_memory_metrics(request_name: str, proc: psutil.Process,
         name=request_name).observe(max(peak_rss - rss_begin, 0))
 
 
-async def execute_request_coroutine(request: api_requests.Request):
+class CoroutineTask:
+    """Wrapper of a background task runs in coroutine"""
+
+    def __init__(self, task: asyncio.Task):
+        self.task = task
+
+    async def cancel(self):
+        try:
+            self.task.cancel()
+            await self.task
+        except asyncio.CancelledError:
+            pass
+
+
+def execute_request_in_coroutine(
+        request: api_requests.Request) -> CoroutineTask:
+    """Execute a request in current event loop.
+
+    Args:
+        request: The request to execute.
+
+    Returns:
+        A CoroutineTask handle to operate the background task.
+    """
+    task = asyncio.create_task(_execute_request_coroutine(request))
+    return CoroutineTask(task)
+
+
+async def _execute_request_coroutine(request: api_requests.Request):
     """Execute a request in current event loop.
 
     Similar to _request_execution_wrapper, but executed as coroutine in current
@@ -640,13 +668,35 @@ def schedule_request(request_id: str,
             The precondition is waited asynchronously and does not block the
             caller.
     """
-    prepare_request(request_id, request_name, request_body, func,
-                    request_cluster_name, schedule_type, is_skypilot_system)
+    request_task = prepare_request(request_id, request_name, request_body, func,
+                                   request_cluster_name, schedule_type,
+                                   is_skypilot_system)
+    schedule_prepared_request(request_task, ignore_return_value, precondition,
+                              retryable)
+
+
+def schedule_prepared_request(request_task: api_requests.Request,
+                              ignore_return_value: bool = False,
+                              precondition: Optional[
+                                  preconditions.Precondition] = None,
+                              retryable: bool = False) -> None:
+    """Enqueue a request to the request queue
+
+    Args:
+        request_task: The prepared request task to schedule.
+        ignore_return_value: If True, the return value of the function will be
+            ignored.
+        precondition: If a precondition is provided, the request will only be
+            scheduled for execution when the precondition is met (returns True).
+            The precondition is waited asynchronously and does not block the
+            caller.
+        retryable: Whether the request should be retried if it fails.
+    """
 
     def enqueue():
-        input_tuple = (request_id, ignore_return_value, retryable)
-        logger.info(f'Queuing request: {request_id}')
-        _get_queue(schedule_type).put(input_tuple)
+        input_tuple = (request_task.request_id, ignore_return_value, retryable)
+        logger.info(f'Queuing request: {request_task.request_id}')
+        _get_queue(request_task.schedule_type).put(input_tuple)
 
     if precondition is not None:
         # Wait async to avoid blocking caller.

sky/server/requests/payloads.py

@@ -316,6 +316,9 @@ class StatusBody(RequestBody):
     all_users: bool = True
     # TODO (kyuds): default to False post 0.10.5
     include_credentials: bool = True
+    # Only return fields that are needed for the
+    # dashboard / CLI summary response
+    summary_response: bool = False
 
 
 class StartBody(RequestBody):
@@ -475,6 +478,17 @@ class VolumeListBody(RequestBody):
     pass
 
 
+class VolumeValidateBody(RequestBody):
+    """The request body for the volume validate endpoint."""
+    name: Optional[str] = None
+    volume_type: Optional[str] = None
+    infra: Optional[str] = None
+    size: Optional[str] = None
+    labels: Optional[Dict[str, str]] = None
+    resource_name: Optional[str] = None
+    config: Optional[Dict[str, Any]] = None
+
+
 class EndpointsBody(RequestBody):
     """The request body for the endpoint."""
     cluster: str
@@ -670,6 +684,13 @@ class LocalUpBody(RequestBody):
     cleanup: bool = False
     context_name: Optional[str] = None
     password: Optional[str] = None
+    name: Optional[str] = None
+    port_start: Optional[int] = None
+
+
+class LocalDownBody(RequestBody):
+    """The request body for the local down endpoint."""
+    name: Optional[str] = None
 
 
 class SSHUpBody(RequestBody):
@@ -709,19 +730,22 @@ class JobsDownloadLogsBody(RequestBody):
 
 class JobsPoolApplyBody(RequestBody):
     """The request body for the jobs pool apply endpoint."""
-    task: str
+    task: Optional[str] = None
+    workers: Optional[int] = None
     pool_name: str
     mode: serve.UpdateMode
 
     def to_kwargs(self) -> Dict[str, Any]:
         kwargs = super().to_kwargs()
-        dag = common.process_mounts_in_task_on_api_server(self.task,
-                                                          self.env_vars,
-                                                          workdir_only=False)
-        assert len(
-            dag.tasks) == 1, ('Must only specify one task in the DAG for '
-                              'a pool.', dag)
-        kwargs['task'] = dag.tasks[0]
+        if self.task is not None:
+            dag = common.process_mounts_in_task_on_api_server(
+                self.task, self.env_vars, workdir_only=False)
+            assert len(
+                dag.tasks) == 1, ('Must only specify one task in the DAG for '
+                                  'a pool.', dag)
+            kwargs['task'] = dag.tasks[0]
+        else:
+            kwargs['task'] = None
         return kwargs
 
 

sky/server/requests/preconditions.py

@@ -146,10 +146,9 @@ class ClusterStartCompletePrecondition(Precondition):
         self.cluster_name = cluster_name
 
     async def check(self) -> Tuple[bool, Optional[str]]:
-        cluster_record = global_user_state.get_cluster_from_name(
+        cluster_status = global_user_state.get_status_from_cluster_name(
             self.cluster_name)
-        if (cluster_record and
-                cluster_record['status'] is status_lib.ClusterStatus.UP):
+        if cluster_status is status_lib.ClusterStatus.UP:
             # Shortcut for started clusters, ignore cluster not found
             # since the cluster record might not yet be created by the
             # launch task.

sky/server/rest.py

@@ -9,6 +9,7 @@ import typing
 from typing import Any, Callable, cast, Optional, TypeVar
 
 import colorama
+import urllib3.exceptions
 
 from sky import exceptions
 from sky import sky_logging
@@ -53,6 +54,7 @@ _session.headers[constants.VERSION_HEADER] = (
 _transient_errors = [
     requests.exceptions.RequestException,
     ConnectionError,
+    urllib3.exceptions.HTTPError,
 ]
 
 

sky/server/server.py

@@ -445,6 +445,22 @@ async def loop_lag_monitor(loop: asyncio.AbstractEventLoop,
     loop.call_at(target, tick)
 
 
+def schedule_on_boot_check():
+    try:
+        executor.schedule_request(
+            request_id='skypilot-server-on-boot-check',
+            request_name='check',
+            request_body=payloads.CheckBody(),
+            func=sky_check.check,
+            schedule_type=requests_lib.ScheduleType.SHORT,
+            is_skypilot_system=True,
+        )
+    except exceptions.RequestAlreadyExistsError:
+        # Lifespan will be executed in each uvicorn worker process, we
+        # can safely ignore the error if the task is already scheduled.
+        logger.debug('Request skypilot-server-on-boot-check already exists.')
+
+
 @contextlib.asynccontextmanager
 async def lifespan(app: fastapi.FastAPI):  # pylint: disable=redefined-outer-name
     """FastAPI lifespan context manager."""
@@ -469,6 +485,7 @@ async def lifespan(app: fastapi.FastAPI):  # pylint: disable=redefined-outer-nam
             # Lifespan will be executed in each uvicorn worker process, we
             # can safely ignore the error if the task is already scheduled.
             logger.debug(f'Request {event.id} already exists.')
+    schedule_on_boot_check()
     asyncio.create_task(cleanup_upload_ids())
     if metrics_utils.METRICS_ENABLED:
         # Start monitoring the event loop lag in each server worker
@@ -1216,19 +1233,8 @@ async def logs(
         schedule_type=requests_lib.ScheduleType.SHORT,
         request_cluster_name=cluster_job_body.cluster_name,
     )
-    task = asyncio.create_task(executor.execute_request_coroutine(request_task))
-
-    async def cancel_task():
-        try:
-            logger.info('Client disconnected for request: '
-                        f'{request.state.request_id}')
-            task.cancel()
-            await task
-        except asyncio.CancelledError:
-            pass
-
-    # Cancel the task after the request is done or client disconnects
-    background_tasks.add_task(cancel_task)
+    task = executor.execute_request_in_coroutine(request_task)
+    background_tasks.add_task(task.cancel)
     # TODO(zhwu): This makes viewing logs in browser impossible. We should adopt
     # the same approach as /stream.
     return stream_utils.stream_response(
@@ -1354,10 +1360,12 @@ def provision_logs(cluster_body: payloads.ClusterNameBody,
     effective_tail = None if tail is None or tail <= 0 else tail
 
     return fastapi.responses.StreamingResponse(
-        content=stream_utils.log_streamer(None,
-                                          log_path,
-                                          tail=effective_tail,
-                                          follow=follow),
+        content=stream_utils.log_streamer(
+            None,
+            log_path,
+            tail=effective_tail,
+            follow=follow,
+            cluster_name=cluster_body.cluster_name),
         media_type='text/plain',
         headers={
             'Cache-Control': 'no-cache, no-transform',
@@ -1419,12 +1427,13 @@ async def local_up(request: fastapi.Request,
 
 
 @app.post('/local_down')
-async def local_down(request: fastapi.Request) -> None:
+async def local_down(request: fastapi.Request,
+                     local_down_body: payloads.LocalDownBody) -> None:
     """Tears down the Kubernetes cluster started by local_up."""
     executor.schedule_request(
         request_id=request.state.request_id,
         request_name='local_down',
-        request_body=payloads.RequestBody(),
+        request_body=local_down_body,
         func=core.local_down,
         schedule_type=requests_lib.ScheduleType.LONG,
     )

sky/server/stream_utils.py

@@ -8,10 +8,12 @@ from typing import AsyncGenerator, Deque, List, Optional
 import aiofiles
 import fastapi
 
+from sky import global_user_state
 from sky import sky_logging
 from sky.server.requests import requests as requests_lib
 from sky.utils import message_utils
 from sky.utils import rich_utils
+from sky.utils import status_lib
 
 logger = sky_logging.init_logger(__name__)
 
@@ -22,6 +24,7 @@ logger = sky_logging.init_logger(__name__)
 _BUFFER_SIZE = 8 * 1024  # 8KB
 _BUFFER_TIMEOUT = 0.02  # 20ms
 _HEARTBEAT_INTERVAL = 30
+_CLUSTER_STATUS_INTERVAL = 1
 
 
 async def _yield_log_file_with_payloads_skipped(
@@ -37,11 +40,13 @@ async def _yield_log_file_with_payloads_skipped(
         yield line_str
 
 
-async def log_streamer(request_id: Optional[str],
-                       log_path: pathlib.Path,
-                       plain_logs: bool = False,
-                       tail: Optional[int] = None,
-                       follow: bool = True) -> AsyncGenerator[str, None]:
+async def log_streamer(
+        request_id: Optional[str],
+        log_path: pathlib.Path,
+        plain_logs: bool = False,
+        tail: Optional[int] = None,
+        follow: bool = True,
+        cluster_name: Optional[str] = None) -> AsyncGenerator[str, None]:
     """Streams the logs of a request.
 
     Args:
@@ -51,6 +56,8 @@ async def log_streamer(request_id: Optional[str],
         plain_logs: Whether to show plain logs.
         tail: The number of lines to tail. If None, tail the whole file.
         follow: Whether to follow the log file.
+        cluster_name: The cluster name to check status for provision logs.
+            If provided and cluster status is UP, streaming will terminate.
     """
 
     if request_id is not None:
@@ -104,15 +111,17 @@ async def log_streamer(request_id: Optional[str],
 
     async with aiofiles.open(log_path, 'rb') as f:
         async for chunk in _tail_log_file(f, request_id, plain_logs, tail,
-                                          follow):
+                                          follow, cluster_name):
             yield chunk
 
 
-async def _tail_log_file(f: aiofiles.threadpool.binary.AsyncBufferedReader,
-                         request_id: Optional[str] = None,
-                         plain_logs: bool = False,
-                         tail: Optional[int] = None,
-                         follow: bool = True) -> AsyncGenerator[str, None]:
+async def _tail_log_file(
+        f: aiofiles.threadpool.binary.AsyncBufferedReader,
+        request_id: Optional[str] = None,
+        plain_logs: bool = False,
+        tail: Optional[int] = None,
+        follow: bool = True,
+        cluster_name: Optional[str] = None) -> AsyncGenerator[str, None]:
     """Tail the opened log file, buffer the lines and flush in chunks."""
 
     if tail is not None:
@@ -128,6 +137,7 @@ async def _tail_log_file(f: aiofiles.threadpool.binary.AsyncBufferedReader,
             yield line_str
 
     last_heartbeat_time = asyncio.get_event_loop().time()
+    last_cluster_status_check_time = asyncio.get_event_loop().time()
 
     # Buffer the lines in memory and flush them in chunks to improve log
     # tailing throughput.
@@ -176,7 +186,19 @@ async def _tail_log_file(f: aiofiles.threadpool.binary.AsyncBufferedReader,
                     break
             if not follow:
                 break
-
+            # Provision logs pass in cluster_name, check cluster status
+            # periodically to see if provisioning is done. We only
+            # check once a second to avoid overloading the DB.
+            check_status = (current_time - last_cluster_status_check_time
+                           ) >= _CLUSTER_STATUS_INTERVAL
+            if cluster_name is not None and check_status:
+                cluster_record = await (
+                    global_user_state.get_status_from_cluster_name_async(
+                        cluster_name))
+                if (cluster_record is None or
+                        cluster_record != status_lib.ClusterStatus.INIT):
+                    break
+                last_cluster_status_check_time = current_time
             if current_time - last_heartbeat_time >= _HEARTBEAT_INTERVAL:
                 # Currently just used to keep the connection busy, refer to
                 # https://github.com/skypilot-org/skypilot/issues/5750 for