skypilot-nightly 1.0.0.dev20250814__py3-none-any.whl → 1.0.0.dev20250816__py3-none-any.whl

sky/serve/service.py

@@ -113,6 +113,9 @@ def cleanup_storage(task_yaml: str) -> bool:
     return not failed
 
 
+# NOTE(dev): We don't need to acquire the `with_lock` in replica manager here
+# because we killed all the processes (controller & replica manager) before
+# calling this function.
 def _cleanup(service_name: str) -> bool:
     """Clean up all service related resources, i.e. replicas and storage."""
     # Cleanup the HA recovery script first as it is possible that some error
@@ -135,28 +138,59 @@ def _cleanup(service_name: str) -> bool:
             continue
         p = multiprocessing.Process(target=replica_managers.terminate_cluster,
                                     args=(info.cluster_name,))
-        p.start()
         info2proc[info] = p
         # Set replica status to `SHUTTING_DOWN`
         info.status_property.sky_launch_status = (
-            replica_managers.ProcessStatus.SUCCEEDED)
+            replica_managers.common_utils.ProcessStatus.SUCCEEDED)
         info.status_property.sky_down_status = (
-            replica_managers.ProcessStatus.RUNNING)
+            replica_managers.common_utils.ProcessStatus.SCHEDULED)
         serve_state.add_or_update_replica(service_name, info.replica_id, info)
-        logger.info(f'Terminating replica {info.replica_id} ...')
-    for info, p in info2proc.items():
-        p.join()
-        if p.exitcode == 0:
-            serve_state.remove_replica(service_name, info.replica_id)
-            logger.info(f'Replica {info.replica_id} terminated successfully.')
-        else:
-            # Set replica status to `FAILED_CLEANUP`
-            info.status_property.sky_down_status = (
-                replica_managers.ProcessStatus.FAILED)
-            serve_state.add_or_update_replica(service_name, info.replica_id,
-                                              info)
-            failed = True
-            logger.error(f'Replica {info.replica_id} failed to terminate.')
+        logger.info(f'Scheduling to terminate replica {info.replica_id} ...')
+
+    def _set_to_failed_cleanup(info: replica_managers.ReplicaInfo) -> None:
+        nonlocal failed
+        # Set replica status to `FAILED_CLEANUP`
+        info.status_property.sky_down_status = (
+            replica_managers.common_utils.ProcessStatus.FAILED)
+        serve_state.add_or_update_replica(service_name, info.replica_id, info)
+        failed = True
+        logger.error(f'Replica {info.replica_id} failed to terminate.')
+
+    # Please reference to sky/serve/replica_managers.py::_refresh_process_pool.
+    # TODO(tian): Refactor to use the same logic and code.
+    while info2proc:
+        snapshot = list(info2proc.items())
+        for info, p in snapshot:
+            if p.is_alive():
+                continue
+            if (info.status_property.sky_down_status ==
+                    replica_managers.common_utils.ProcessStatus.SCHEDULED):
+                if controller_utils.can_terminate():
+                    try:
+                        p.start()
+                    except Exception as e:  # pylint: disable=broad-except
+                        _set_to_failed_cleanup(info)
+                        logger.error(f'Failed to start process for replica '
+                                     f'{info.replica_id}: {e}')
+                        del info2proc[info]
+                    else:
+                        info.status_property.sky_down_status = (
+                            common_utils.ProcessStatus.RUNNING)
+                        serve_state.add_or_update_replica(
+                            service_name, info.replica_id, info)
+            else:
+                logger.info('Terminate process for replica '
+                            f'{info.replica_id} finished.')
+                p.join()
+                del info2proc[info]
+                if p.exitcode == 0:
+                    serve_state.remove_replica(service_name, info.replica_id)
+                    logger.info(
+                        f'Replica {info.replica_id} terminated successfully.')
+                else:
+                    _set_to_failed_cleanup(info)
+        time.sleep(3)
+
     versions = serve_state.get_service_versions(service_name)
     serve_state.remove_service_versions(service_name)
 

sky/server/common.py

@@ -5,7 +5,6 @@ import enum
 import functools
 from http.cookiejar import CookieJar
 from http.cookiejar import MozillaCookieJar
-import json
 import os
 import pathlib
 import re
@@ -372,7 +371,7 @@ def _handle_non_200_server_status(
                          '') == ApiServerStatus.VERSION_MISMATCH.value):
                 return ApiServerInfo(status=ApiServerStatus.VERSION_MISMATCH,
                                      error=body.get('message', ''))
-        except json.JSONDecodeError:
+        except requests.JSONDecodeError:
             pass
     return ApiServerInfo(status=ApiServerStatus.UNHEALTHY)
 
@@ -463,7 +462,7 @@ def get_api_server_status(endpoint: Optional[str] = None) -> ApiServerInfo:
             # OAuth.
             set_api_cookie_jar(cookies, create_if_not_exists=True)
             return server_info
-        except (json.JSONDecodeError, AttributeError) as e:
+        except (requests.JSONDecodeError, AttributeError) as e:
             # Try to check if we got redirected to a login page.
             for prev_response in response.history:
                 logger.debug(f'Previous response: {prev_response.url}')

sky/server/constants.py

@@ -10,7 +10,7 @@ from sky.skylet import constants
 # based on version info is needed.
 # For more details and code guidelines, refer to:
 # https://docs.skypilot.co/en/latest/developers/CONTRIBUTING.html#backward-compatibility-guidelines
-API_VERSION = 16
+API_VERSION = 17
 
 # The minimum peer API version that the code should still work with.
 # Notes (dev):

sky/server/requests/payloads.py

@@ -497,6 +497,12 @@ class JobsQueueBody(RequestBody):
     skip_finished: bool = False
     all_users: bool = False
     job_ids: Optional[List[int]] = None
+    user_match: Optional[str] = None
+    workspace_match: Optional[str] = None
+    name_match: Optional[str] = None
+    pool_match: Optional[str] = None
+    page: Optional[int] = None
+    limit: Optional[int] = None
 
 
 class JobsCancelBody(RequestBody):

sky/server/requests/serializers/decoders.py

@@ -9,6 +9,7 @@ from sky import models
 from sky.catalog import common
 from sky.data import storage
 from sky.provision.kubernetes import utils as kubernetes_utils
+from sky.schemas.api import responses
 from sky.serve import serve_state
 from sky.server import constants as server_constants
 from sky.skylet import job_lib
@@ -50,13 +51,17 @@ def default_decode_handler(return_value: Any) -> Any:
 
 
 @register_decoders('status')
-def decode_status(return_value: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+def decode_status(
+        return_value: List[Dict[str, Any]]) -> List[responses.StatusResponse]:
     clusters = return_value
+    response = []
     for cluster in clusters:
         cluster['handle'] = decode_and_unpickle(cluster['handle'])
         cluster['status'] = status_lib.ClusterStatus(cluster['status'])
-
-    return clusters
+        cluster['storage_mounts_metadata'] = decode_and_unpickle(
+            cluster['storage_mounts_metadata'])
+        response.append(responses.StatusResponse.model_validate(cluster))
+    return response
 
 
 @register_decoders('status_kubernetes')
@@ -102,8 +107,18 @@ def decode_queue(return_value: List[dict],) -> List[Dict[str, Any]]:
 
 
 @register_decoders('jobs.queue')
-def decode_jobs_queue(return_value: List[dict],) -> List[Dict[str, Any]]:
-    jobs = return_value
+def decode_jobs_queue(return_value):
+    """Decode jobs queue response.
+
+    Supports legacy list, or a dict {jobs, total}.
+    - Returns list[job]
+    """
+    # Case 1: dict shape {jobs, total}
+    if isinstance(return_value, dict) and 'jobs' in return_value:
+        jobs = return_value.get('jobs', [])
+    else:
+        # Case 2: legacy list
+        jobs = return_value
     for job in jobs:
         job['status'] = managed_jobs.ManagedJobStatus(job['status'])
     return jobs

sky/server/requests/serializers/encoders.py

@@ -8,6 +8,7 @@ import pickle
 import typing
 from typing import Any, Dict, List, Optional, Tuple
 
+from sky.schemas.api import responses
 from sky.server import constants as server_constants
 
 if typing.TYPE_CHECKING:
@@ -51,13 +52,17 @@ def default_encoder(return_value: Any) -> Any:
 
 
 @register_encoder('status')
-def encode_status(clusters: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+def encode_status(
+        clusters: List[responses.StatusResponse]) -> List[Dict[str, Any]]:
+    response = []
     for cluster in clusters:
-        cluster['status'] = cluster['status'].value
-        cluster['handle'] = pickle_and_encode(cluster['handle'])
-        cluster['storage_mounts_metadata'] = pickle_and_encode(
-            cluster['storage_mounts_metadata'])
-    return clusters
+        response_cluster = cluster.model_dump()
+        response_cluster['status'] = cluster['status'].value
+        response_cluster['handle'] = pickle_and_encode(cluster['handle'])
+        response_cluster['storage_mounts_metadata'] = pickle_and_encode(
+            response_cluster['storage_mounts_metadata'])
+        response.append(response_cluster)
+    return response
 
 
 @register_encoder('launch', 'exec', 'jobs.launch')
@@ -106,10 +111,18 @@ def encode_status_kubernetes(
 
 
 @register_encoder('jobs.queue')
-def encode_jobs_queue(jobs: List[dict],) -> List[Dict[str, Any]]:
+def encode_jobs_queue(jobs_or_tuple):
+    # Support returning either a plain jobs list or a (jobs, total) tuple
+    if isinstance(jobs_or_tuple, tuple) and len(jobs_or_tuple) == 2:
+        jobs, total = jobs_or_tuple
+    else:
+        jobs = jobs_or_tuple
+        total = None
     for job in jobs:
         job['status'] = job['status'].value
-    return jobs
+    if total is None:
+        return jobs
+    return {'jobs': jobs, 'total': total}
 
 
 def _encode_serve_status(

sky/server/server.py

@@ -792,8 +792,6 @@ async def validate(validate_body: payloads.ValidateBody) -> None:
     ctx.override_envs(validate_body.env_vars)
 
     def validate_dag(dag: dag_utils.dag_lib.Dag):
-        # Resolve the volumes before admin policy and validation.
-        dag.resolve_and_validate_volumes()
         # TODO: Admin policy may contain arbitrary code, which may be expensive
         # to run and may block the server thread. However, moving it into the
         # executor adds a ~150ms penalty on the local API server because of
@@ -802,6 +800,7 @@ async def validate(validate_body: payloads.ValidateBody) -> None:
         with admin_policy_utils.apply_and_use_config_in_current_request(
                 dag,
                 request_options=validate_body.get_request_options()) as dag:
+            dag.resolve_and_validate_volumes()
             # Skip validating workdir and file_mounts, as those need to be
             # validated after the files are uploaded to the SkyPilot API server
             # with `upload_mounts_to_api_server`.
@@ -1233,7 +1232,8 @@ async def download_logs(
 
 
 @app.post('/download')
-async def download(download_body: payloads.DownloadBody) -> None:
+async def download(download_body: payloads.DownloadBody,
+                   request: fastapi.Request) -> None:
     """Downloads a folder from the cluster to the local machine."""
     folder_paths = [
         pathlib.Path(folder_path) for folder_path in download_body.folder_paths
@@ -1262,7 +1262,16 @@ async def download(download_body: payloads.DownloadBody) -> None:
             str(folder_path.expanduser().resolve())
             for folder_path in folder_paths
         ]
-        storage_utils.zip_files_and_folders(folders, zip_path)
+        # Check for optional query parameter to control zip entry structure
+        relative = request.query_params.get('relative', 'home')
+        if relative == 'items':
+            # Dashboard-friendly: entries relative to selected folders
+            storage_utils.zip_files_and_folders(folders,
+                                                zip_path,
+                                                relative_to_items=True)
+        else:
+            # CLI-friendly (default): entries with full paths for mapping
+            storage_utils.zip_files_and_folders(folders, zip_path)
 
         # Add home path to the response headers, so that the client can replace
         # the remote path in the zip file to the local path.
@@ -1284,6 +1293,46 @@ async def download(download_body: payloads.DownloadBody) -> None:
                                     detail=f'Error creating zip file: {str(e)}')
 
 
+@app.post('/provision_logs')
+async def provision_logs(cluster_body: payloads.ClusterNameBody,
+                         follow: bool = True,
+                         tail: int = 0) -> fastapi.responses.StreamingResponse:
+    """Streams the provision.log for the latest launch request of a cluster."""
+    # Prefer clusters table first, then cluster_history as fallback.
+    log_path_str = global_user_state.get_cluster_provision_log_path(
+        cluster_body.cluster_name)
+    if not log_path_str:
+        log_path_str = global_user_state.get_cluster_history_provision_log_path(
+            cluster_body.cluster_name)
+    if not log_path_str:
+        raise fastapi.HTTPException(
+            status_code=404,
+            detail=('Provision log path is not recorded for this cluster. '
+                    'Please relaunch to generate provisioning logs.'))
+
+    log_path = pathlib.Path(log_path_str).expanduser().resolve()
+    if not log_path.exists():
+        raise fastapi.HTTPException(
+            status_code=404,
+            detail=f'Provision log path does not exist: {str(log_path)}')
+
+    # Tail semantics: 0 means print all lines. Convert 0 -> None for streamer.
+    effective_tail = None if tail is None or tail <= 0 else tail
+
+    return fastapi.responses.StreamingResponse(
+        content=stream_utils.log_streamer(None,
+                                          log_path,
+                                          tail=effective_tail,
+                                          follow=follow),
+        media_type='text/plain',
+        headers={
+            'Cache-Control': 'no-cache, no-transform',
+            'X-Accel-Buffering': 'no',
+            'Transfer-Encoding': 'chunked',
+        },
+    )
+
+
 @app.post('/cost_report')
 async def cost_report(request: fastapi.Request,
                       cost_report_body: payloads.CostReportBody) -> None:
@@ -1541,13 +1590,7 @@ async def health(request: fastapi.Request) -> responses.APIHealthResponse:
     """Checks the health of the API server.
 
     Returns:
-        A dictionary with the following keys:
-        - status: str; The status of the API server.
-        - api_version: str; The API version of the API server.
-        - version: str; The version of SkyPilot used for API server.
-        - version_on_disk: str; The version of the SkyPilot installation on
-          disk, which can be used to warn about restarting the API server
-        - commit: str; The commit hash of SkyPilot used for API server.
+        responses.APIHealthResponse: The health response.
     """
     user = request.state.auth_user
     server_status = common.ApiServerStatus.HEALTHY
@@ -1815,6 +1858,9 @@ if __name__ == '__main__':
             global_tasks.append(background.create_task(metrics_server.serve()))
         global_tasks.append(
             background.create_task(requests_lib.requests_gc_daemon()))
+        global_tasks.append(
+            background.create_task(
+                global_user_state.cluster_event_retention_daemon()))
         threading.Thread(target=background.run_forever, daemon=True).start()
 
         queue_server, workers = executor.start(config)

sky/templates/kubernetes-ray.yml.j2

@@ -378,6 +378,7 @@ available_node_types:
         {% if volume_mounts %}
         securityContext:
           fsGroup: 1000
+          fsGroupChangePolicy: OnRootMismatch
         {% endif %}
 
         # Add node selector if GPU/TPUs are requested:

sky/utils/cli_utils/status_utils.py

@@ -6,6 +6,7 @@ import click
 import colorama
 
 from sky import backends
+from sky.schemas.api import responses
 from sky.utils import common_utils
 from sky.utils import log_utils
 from sky.utils import resources_utils
@@ -44,7 +45,7 @@ class StatusColumn:
         return val
 
 
-def show_status_table(cluster_records: List[_ClusterRecord],
+def show_status_table(cluster_records: List[responses.StatusResponse],
                       show_all: bool,
                       show_user: bool,
                       query_clusters: Optional[List[str]] = None,

sky/utils/common_utils.py

@@ -1,6 +1,7 @@
 """Utils shared between all of sky"""
 
 import difflib
+import enum
 import functools
 import getpass
 import hashlib
@@ -55,6 +56,25 @@ _VALID_ENV_VAR_REGEX = '[a-zA-Z_][a-zA-Z0-9_]*'
 logger = sky_logging.init_logger(__name__)
 
 
+class ProcessStatus(enum.Enum):
+    """Process status."""
+
+    # The process is scheduled to run, but not started yet.
+    SCHEDULED = 'SCHEDULED'
+
+    # The process is running
+    RUNNING = 'RUNNING'
+
+    # The process is finished and succeeded
+    SUCCEEDED = 'SUCCEEDED'
+
+    # The process is interrupted
+    INTERRUPTED = 'INTERRUPTED'
+
+    # The process failed
+    FAILED = 'FAILED'
+
+
 @annotations.lru_cache(scope='request')
 def get_usage_run_id() -> str:
     """Returns a unique run id for each 'run'.

sky/utils/controller_utils.py

@@ -1224,13 +1224,26 @@ def _get_launch_parallelism() -> int:
 
 
 def can_provision() -> bool:
-    num_provision = (
-        serve_state.total_number_provisioning_replicas() * SERVE_LAUNCH_RATIO +
-        managed_job_state.get_num_launching_jobs())
-    return num_provision < _get_launch_parallelism()
+    # We always prioritize terminating over provisioning, to save the cost on
+    # idle resources.
+    if serve_state.total_number_scheduled_to_terminate_replicas() > 0:
+        return False
+    return can_terminate()
 
 
 def can_start_new_process() -> bool:
     num_procs = (serve_state.get_num_services() * SERVE_PROC_RATIO +
                  managed_job_state.get_num_alive_jobs())
     return num_procs < _get_job_parallelism()
+
+
+# We limit the number of terminating replicas to the number of CPUs. This is
+# just a temporary solution to avoid overwhelming the controller. After one job
+# controller PR, we should use API server to handle resources management.
+def can_terminate() -> bool:
+    num_terminating = (
+        serve_state.total_number_provisioning_replicas() * SERVE_LAUNCH_RATIO +
+        # Each terminate process will take roughly the same CPUs as job launch.
+        serve_state.total_number_terminating_replicas() +
+        managed_job_state.get_num_launching_jobs())
+    return num_terminating < _get_launch_parallelism()

sky/utils/db/migration_utils.py

@@ -19,7 +19,7 @@ logger = sky_logging.init_logger(__name__)
 DB_INIT_LOCK_TIMEOUT_SECONDS = 10
 
 GLOBAL_USER_STATE_DB_NAME = 'state_db'
-GLOBAL_USER_STATE_VERSION = '005'
+GLOBAL_USER_STATE_VERSION = '006'
 GLOBAL_USER_STATE_LOCK_PATH = '~/.sky/locks/.state_db.lock'
 
 SPOT_JOBS_DB_NAME = 'spot_jobs_db'

sky/utils/log_utils.py

@@ -47,13 +47,16 @@ class RayUpLineProcessor(LineProcessor):
         RUNTIME_SETUP = 1
         PULLING_DOCKER_IMAGES = 2
 
-    def __init__(self, log_path: str):
+    def __init__(self, log_path: str, cluster_name: Optional[str] = None):
         self.log_path = log_path
+        self.cluster_name = cluster_name
 
     def __enter__(self) -> None:
         self.state = self.ProvisionStatus.LAUNCH
         self.status_display = rich_utils.safe_status(
-            ux_utils.spinner_message('Launching', self.log_path))
+            ux_utils.spinner_message('Launching',
+                                     self.log_path,
+                                     cluster_name=self.cluster_name))
         self.status_display.start()
 
     def process_line(self, log_line: str) -> None:
@@ -62,19 +65,25 @@ class RayUpLineProcessor(LineProcessor):
             logger.info('  Head VM is up.')
             self.status_display.update(
                 ux_utils.spinner_message(
-                    'Launching - Preparing SkyPilot runtime', self.log_path))
+                    'Launching - Preparing SkyPilot runtime',
+                    self.log_path,
+                    cluster_name=self.cluster_name))
             self.state = self.ProvisionStatus.RUNTIME_SETUP
         if ('Pulling from' in log_line and
                 self.state == self.ProvisionStatus.RUNTIME_SETUP):
             self.status_display.update(
                 ux_utils.spinner_message(
-                    'Launching - Initializing docker container', self.log_path))
+                    'Launching - Initializing docker container',
+                    self.log_path,
+                    cluster_name=self.cluster_name))
             self.state = self.ProvisionStatus.PULLING_DOCKER_IMAGES
         if ('Status: Downloaded newer image' in log_line and
                 self.state == self.ProvisionStatus.PULLING_DOCKER_IMAGES):
             self.status_display.update(
                 ux_utils.spinner_message(
-                    'Launching - Preparing SkyPilot runtime', self.log_path))
+                    'Launching - Preparing SkyPilot runtime',
+                    self.log_path,
+                    cluster_name=self.cluster_name))
             self.state = self.ProvisionStatus.RUNTIME_SETUP
 
     def __exit__(self, except_type: Optional[Type[BaseException]],

sky/utils/resources_utils.py

@@ -5,7 +5,7 @@ import itertools
 import json
 import math
 import typing
-from typing import Dict, List, Optional, Set, Union
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
 
 from sky import skypilot_config
 from sky.skylet import constants
@@ -435,3 +435,27 @@ def parse_time_minutes(time: str) -> int:
                 continue
 
     raise ValueError(f'Invalid time format: {time}')
+
+
+def normalize_any_of_resources_config(
+        any_of: List[Dict[str, Any]]) -> Tuple[str, ...]:
+    """Normalize a list of any_of resources config to a canonical form.
+
+    Args:
+        any_of: A list of any_of resources config.
+
+    Returns:
+        A normalized tuple representation that can be compared for equality.
+        Two lists with the same resource configurations in different orders
+        will produce the same normalized result.
+    """
+    if not any_of:
+        return tuple()
+
+    # Convert each config to JSON string with sorted keys, then sort the list
+    normalized_configs = [
+        json.dumps(config, sort_keys=True, separators=(',', ':'))
+        for config in any_of
+    ]
+
+    return tuple(sorted(normalized_configs))

sky/utils/schemas.py

@@ -1535,6 +1535,9 @@ def get_config_schema():
             'requests_retention_hours': {
                 'type': 'integer',
             },
+            'cluster_event_retention_hours': {
+                'type': 'number',
+            },
         }
     }
 

sky/utils/ux_utils.py

@@ -26,9 +26,16 @@ BOLD = '\033[1m'
 RESET_BOLD = '\033[0m'
 
 # Log path hint in the spinner during launching
+# (old, kept for backward compatibility)
 _LOG_PATH_HINT = (f'{colorama.Style.DIM}View logs: sky api logs -l '
                   '{log_path}'
                   f'{colorama.Style.RESET_ALL}')
+# Log hint: recommend sky logs --provision <cluster_name>
+_PROVISION_LOG_HINT = (
+    f'{colorama.Style.DIM}View logs: '
+    f'{BOLD}sky logs --provision {{cluster_name}}{RESET_BOLD}'
+    f'{colorama.Style.RESET_ALL}')
+# Legacy path hint retained for local-only cases where we don't have cluster
 _LOG_PATH_HINT_LOCAL = (f'{colorama.Style.DIM}View logs: '
                         '{log_path}'
                         f'{colorama.Style.RESET_ALL}')
@@ -126,7 +133,10 @@ class RedirectOutputForProcess:
 
 def log_path_hint(log_path: Union[str, 'pathlib.Path'],
                   is_local: bool = False) -> str:
-    """Gets the log path hint for the given log path."""
+    """Gets the log path hint for the given log path.
+
+    Kept for backward compatibility when only paths are available.
+    """
     log_path = str(log_path)
     expanded_home = os.path.expanduser('~')
     if log_path.startswith(expanded_home):
@@ -139,6 +149,12 @@ def log_path_hint(log_path: Union[str, 'pathlib.Path'],
     return _LOG_PATH_HINT.format(log_path=log_path)
 
 
+def provision_hint(cluster_name: Optional[str]) -> Optional[str]:
+    if not cluster_name:
+        return None
+    return _PROVISION_LOG_HINT.format(cluster_name=cluster_name)
+
+
 def starting_message(message: str) -> str:
     """Gets the starting message for the given message."""
     # We have to reset the color before the message, because sometimes if a
@@ -150,7 +166,8 @@ def starting_message(message: str) -> str:
 def finishing_message(message: str,
                       log_path: Optional[Union[str, 'pathlib.Path']] = None,
                       is_local: bool = False,
-                      follow_up_message: Optional[str] = None) -> str:
+                      follow_up_message: Optional[str] = None,
+                      cluster_name: Optional[str] = None) -> str:
     """Gets the finishing message for the given message.
 
     Args:
@@ -168,6 +185,9 @@ def finishing_message(message: str,
     success_prefix = (f'{colorama.Style.RESET_ALL}{colorama.Fore.GREEN}✓ '
                       f'{message}{colorama.Style.RESET_ALL}{follow_up_message}'
                       f'{colorama.Style.RESET_ALL}')
+    hint = provision_hint(cluster_name)
+    if hint:
+        return f'{success_prefix}  {hint}'
     if log_path is None:
         return success_prefix
     path_hint = log_path_hint(log_path, is_local)
@@ -176,13 +196,17 @@ def finishing_message(message: str,
 
 def error_message(message: str,
                   log_path: Optional[Union[str, 'pathlib.Path']] = None,
-                  is_local: bool = False) -> str:
+                  is_local: bool = False,
+                  cluster_name: Optional[str] = None) -> str:
     """Gets the error message for the given message."""
     # We have to reset the color before the message, because sometimes if a
     # previous spinner with dimmed color overflows in a narrow terminal, the
     # color might be messed up.
     error_prefix = (f'{colorama.Style.RESET_ALL}{colorama.Fore.RED}⨯'
                     f'{colorama.Style.RESET_ALL} {message}')
+    hint = provision_hint(cluster_name)
+    if hint:
+        return f'{error_prefix}  {hint}'
     if log_path is None:
         return error_prefix
     path_hint = log_path_hint(log_path, is_local)
@@ -200,9 +224,16 @@ def retry_message(message: str) -> str:
 
 def spinner_message(message: str,
                     log_path: Optional[Union[str, 'pathlib.Path']] = None,
-                    is_local: bool = False) -> str:
-    """Gets the spinner message for the given message and log path."""
+                    is_local: bool = False,
+                    cluster_name: Optional[str] = None) -> str:
+    """Gets the spinner message for the given message and log path.
+
+    If cluster_name is provided, recommend `sky logs --provision <cluster>`.
+    """
     colored_spinner = f'[bold cyan]{message}[/]'
+    hint = provision_hint(cluster_name)
+    if hint:
+        return f'{colored_spinner}  {hint}'
     if log_path is None:
         return colored_spinner
     path_hint = log_path_hint(log_path, is_local)

{skypilot_nightly-1.0.0.dev20250814.dist-info → skypilot_nightly-1.0.0.dev20250816.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skypilot-nightly
-Version: 1.0.0.dev20250814
+Version: 1.0.0.dev20250816
 Summary: SkyPilot: Run AI on Any Infra — Unified, Faster, Cheaper.
 Author: SkyPilot Team
 License: Apache 2.0