skypilot-nightly 1.0.0.dev20250702__py3-none-any.whl → 1.0.0.dev20250704__py3-none-any.whl

sky/server/rest.py

@@ -12,6 +12,8 @@ import colorama
 from sky import exceptions
 from sky import sky_logging
 from sky.adaptors import common as adaptors_common
+from sky.server import constants
+from sky.server import versions
 from sky.utils import common_utils
 from sky.utils import rich_utils
 from sky.utils import ux_utils
@@ -28,6 +30,11 @@ F = TypeVar('F', bound=Callable[..., Any])
 
 _RETRY_CONTEXT = contextvars.ContextVar('retry_context', default=None)
 
+_session = requests.Session()
+_session.headers[constants.API_VERSION_HEADER] = str(constants.API_VERSION)
+_session.headers[constants.VERSION_HEADER] = (
+    versions.get_local_readable_version())
+
 
 class RetryContext:
 
@@ -132,13 +139,17 @@ def handle_server_unavailable(response: 'requests.Response') -> None:
 def request(method, url, **kwargs) -> 'requests.Response':
     """Send a request to the API server, retry on server temporarily
     unavailable."""
-    response = requests.request(method, url, **kwargs)
-    handle_server_unavailable(response)
-    return response
+    return request_without_retry(method, url, **kwargs)
 
 
 def request_without_retry(method, url, **kwargs) -> 'requests.Response':
     """Send a request to the API server without retry."""
-    response = requests.request(method, url, **kwargs)
+    response = _session.request(method, url, **kwargs)
     handle_server_unavailable(response)
+    remote_api_version = response.headers.get(constants.API_VERSION_HEADER)
+    remote_version = response.headers.get(constants.VERSION_HEADER)
+    if remote_api_version is not None:
+        versions.set_remote_api_version(int(remote_api_version))
+    if remote_version is not None:
+        versions.set_remote_version(remote_version)
     return response

sky/server/server.py

@@ -4,7 +4,6 @@ import argparse
 import asyncio
 import base64
 import contextlib
-import dataclasses
 import datetime
 import hashlib
 import json
@@ -14,6 +13,7 @@ import os
 import pathlib
 import posixpath
 import re
+import resource
 import shutil
 import sys
 import threading
@@ -48,6 +48,7 @@ from sky.server import constants as server_constants
 from sky.server import metrics
 from sky.server import state
 from sky.server import stream_utils
+from sky.server import versions
 from sky.server.requests import executor
 from sky.server.requests import payloads
 from sky.server.requests import preconditions
@@ -558,6 +559,35 @@ class GracefulShutdownMiddleware(starlette.middleware.base.BaseHTTPMiddleware):
         return await call_next(request)
 
 
+class APIVersionMiddleware(starlette.middleware.base.BaseHTTPMiddleware):
+    """Middleware to add API version to the request."""
+
+    async def dispatch(self, request: fastapi.Request, call_next):
+        version_info = versions.check_compatibility_at_server(request.headers)
+        # Bypass version handling for backward compatibility with clients prior
+        # to v0.11.0, the client will check the version in the body of
+        # /api/health response and hint an upgrade.
+        # TODO(aylei): remove this after v0.13.0 is released.
+        if version_info is None:
+            return await call_next(request)
+        if version_info.error is None:
+            versions.set_remote_api_version(version_info.api_version)
+            versions.set_remote_version(version_info.version)
+            response = await call_next(request)
+        else:
+            response = fastapi.responses.JSONResponse(
+                status_code=400,
+                content={
+                    'error': common.ApiServerStatus.VERSION_MISMATCH.value,
+                    'message': version_info.error,
+                })
+        response.headers[server_constants.API_VERSION_HEADER] = str(
+            server_constants.API_VERSION)
+        response.headers[server_constants.VERSION_HEADER] = \
+            versions.get_local_readable_version()
+        return response
+
+
 app = fastapi.FastAPI(prefix='/api/v1', debug=True, lifespan=lifespan)
 # Middleware wraps in the order defined here. E.g., given
 #   app.add_middleware(Middleware1)
@@ -570,6 +600,8 @@ app = fastapi.FastAPI(prefix='/api/v1', debug=True, lifespan=lifespan)
 # Use environment variable to make the metrics middleware optional.
 if os.environ.get(constants.ENV_VAR_SERVER_METRICS_ENABLED):
     app.add_middleware(metrics.PrometheusMiddleware)
+app.add_middleware(APIVersionMiddleware)
+app.add_middleware(RBACMiddleware)
 app.add_middleware(InternalDashboardPrefixMiddleware)
 app.add_middleware(GracefulShutdownMiddleware)
 app.add_middleware(PathCleanMiddleware)
@@ -614,6 +646,16 @@ app.include_router(ssh_node_pools_rest.router,
                    prefix='/ssh_node_pools',
                    tags=['ssh_node_pools'])
 
+# Increase the limit of files we can open to our hard limit. This fixes bugs
+# where we can not aquire file locks or open enough logs and the API server
+# crashes. On Mac, the hard limit is 9,223,372,036,854,775,807.
+# TODO(luca) figure out what to do if we need to open more than 2^63 files.
+try:
+    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+    resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard))
+except Exception:  # pylint: disable=broad-except
+    pass  # no issue, we will warn the user later if its too low
+
 
 @app.get('/token')
 async def token(request: fastapi.Request,
@@ -1324,7 +1366,7 @@ async def local_down(request: fastapi.Request) -> None:
 
 # === API server related APIs ===
 @app.get('/api/get')
-async def api_get(request_id: str) -> requests_lib.RequestPayload:
+async def api_get(request_id: str) -> payloads.RequestPayload:
     """Gets a request with a given request ID prefix."""
     while True:
         request_task = requests_lib.get_request(request_id)
@@ -1339,9 +1381,8 @@ async def api_get(request_id: str) -> requests_lib.RequestPayload:
                     detail=f'Request {request_id!r} should be retried')
             request_error = request_task.get_error()
             if request_error is not None:
-                raise fastapi.HTTPException(status_code=500,
-                                            detail=dataclasses.asdict(
-                                                request_task.encode()))
+                raise fastapi.HTTPException(
+                    status_code=500, detail=request_task.encode().model_dump())
             return request_task.encode()
         # yield control to allow other coroutines to run, sleep shortly
         # to avoid storming the DB and CPU in the meantime
@@ -1479,7 +1520,7 @@ async def api_status(
         None, description='Request IDs to get status for.'),
     all_status: bool = fastapi.Query(
         False, description='Get finished requests as well.'),
-) -> List[requests_lib.RequestPayload]:
+) -> List[payloads.RequestPayload]:
     """Gets the list of requests."""
     if request_ids is None:
         statuses = None
@@ -1519,7 +1560,10 @@ async def health(request: fastapi.Request) -> Dict[str, Any]:
     logger.info(f'Health endpoint: request.state.auth_user = {user}')
     return {
         'status': common.ApiServerStatus.HEALTHY.value,
-        'api_version': server_constants.API_VERSION,
+        # Kept for backward compatibility, clients before 0.11.0 will read this
+        # field to check compatibility and hint the user to upgrade the CLI.
+        # TODO(aylei): remove this field after 0.13.0
+        'api_version': str(server_constants.API_VERSION),
         'version': sky.__version__,
         'version_on_disk': common.get_skypilot_version_on_disk(),
         'commit': sky.__commit__,

sky/server/versions.py

@@ -0,0 +1,270 @@
+"""API versioning module."""
+
+import contextvars
+import functools
+import re
+from typing import Callable, Literal, Mapping, NamedTuple, Optional, Tuple
+
+import colorama
+from packaging import version as version_lib
+
+import sky
+from sky import exceptions
+from sky import sky_logging
+from sky.server import constants
+from sky.utils import ux_utils
+
+logger = sky_logging.init_logger(__name__)
+
+CLIENT_TOO_OLD_ERROR = (
+    f'{colorama.Fore.YELLOW}Your SkyPilot client version is too old:'
+    '{remote_version}\n'
+    f'{colorama.Style.RESET_ALL}'
+    'The server is running on {local_version} and the minimum compatible '
+    'version is {min_version}.\n'
+    f'Upgrade your client with:\n{colorama.Fore.YELLOW}'
+    '{command}'
+    f'{colorama.Style.RESET_ALL}')
+SERVER_TOO_OLD_ERROR = (
+    f'{colorama.Fore.YELLOW}Your SkyPilot API server version is too old: '
+    '{remote_version}\n'
+    f'{colorama.Style.RESET_ALL}'
+    'The client is running on {local_version} and the minimum compatible '
+    'version is {min_version}.\n'
+    'Contact your administrator to upgrade the remote API server or downgrade '
+    f'your client with:\n{colorama.Fore.YELLOW}'
+    '{command}'
+    f'{colorama.Style.RESET_ALL}')
+
+# SkyPilot dev version.
+DEV_VERSION = '1.0.0-dev0'
+
+_REMOTE_TO_ERROR = {
+    'client': CLIENT_TOO_OLD_ERROR,
+    'server': SERVER_TOO_OLD_ERROR,
+}
+
+# Context-local (thread or cooroutine) remote API version, captured during
+# communication with the remote peer.
+_remote_api_version: contextvars.ContextVar[Optional[int]] = \
+    contextvars.ContextVar('remote_api_version', default=None)
+_remote_version: contextvars.ContextVar[str] = \
+    contextvars.ContextVar('remote_version', default='unknown')
+_reminded_for_minor_version_upgrade = False
+
+
+def get_remote_api_version() -> Optional[int]:
+    return _remote_api_version.get()
+
+
+def set_remote_api_version(api_version: int) -> None:
+    _remote_api_version.set(api_version)
+
+
+def get_remote_version() -> str:
+    return _remote_version.get()
+
+
+def set_remote_version(version: str) -> None:
+    _remote_version.set(version)
+
+
+class VersionInfo(NamedTuple):
+    api_version: int
+    version: str
+    error: Optional[str] = None
+
+
+def check_compatibility_at_server(
+        client_headers: Mapping[str, str]) -> Optional[VersionInfo]:
+    """Check API compatibility between client and server."""
+    return _check_version_compatibility(client_headers, 'client')
+
+
+def check_compatibility_at_client(
+        server_headers: Mapping[str, str]) -> Optional[VersionInfo]:
+    """Check API compatibility between client and server."""
+    return _check_version_compatibility(server_headers, 'server')
+
+
+def _check_version_compatibility(
+        remote_headers: Mapping[str, str],
+        remote_type: Literal['client', 'server']) -> Optional[VersionInfo]:
+    """Check API compatibility between client and server.
+
+    This function can be called at both client and server side, where the
+    headers should contain the version info of the remote.
+
+    Args:
+        remote_headers: The headers of the request/response sent from the
+            remote.
+        remote_type: The type of the remote, used to determine the error
+            message. Valid options are 'client' and 'server'.
+
+    Returns:
+        The version info of the remote, None if the version info is not found
+        in the headers for backward compatibility.
+    """
+    api_version_str = remote_headers.get(constants.API_VERSION_HEADER)
+    version = remote_headers.get(constants.VERSION_HEADER)
+    if version is None or api_version_str is None:
+        return None
+    try:
+        api_version = int(api_version_str)
+    except ValueError:
+        # The future change is expected to not break the compatibility of this
+        # header, so we are encountering a bug or a malicious request here,
+        # just raise an error.
+        raise ValueError(
+            f'Header {constants.API_VERSION_HEADER}: '
+            f'{api_version_str} is not a valid API version.') from None
+
+    if api_version < constants.MIN_COMPATIBLE_API_VERSION:
+        if remote_type == 'server':
+            # Hint the user to downgrade to client to the remote server server.
+            server_version, server_commit = parse_readable_version(version)
+            command = install_version_command(server_version, server_commit)
+        else:
+            # Hint the client to upgrade to upgrade the server version
+            command = install_version_command(sky.__version__, sky.__commit__)
+        return VersionInfo(api_version=api_version,
+                           version=version,
+                           error=_REMOTE_TO_ERROR[remote_type].format(
+                               remote_version=version,
+                               local_version=get_local_readable_version(),
+                               min_version=constants.MIN_COMPATIBLE_VERSION,
+                               command=command,
+                           ))
+
+    if remote_type == 'server':
+        # Only print the reminder at client-side.
+        _remind_minor_version_upgrade(version)
+
+    return VersionInfo(api_version=api_version, version=version)
+
+
+def get_local_readable_version() -> str:
+    """Get the readable version of the SkyPilot code loaded in current process.
+
+    For dev version, the version is formatted as: 1.0.0-dev0 (commit: 1234567)
+    to make it meaningful for users.
+    """
+    if sky.__version__ == DEV_VERSION:
+        return f'{sky.__version__} (commit: {sky.__commit__})'
+    else:
+        return sky.__version__
+
+
+def parse_readable_version(version: str) -> Tuple[str, Optional[str]]:
+    """Parse a readable produced by get_local_readable_version.
+
+    Args:
+        version: The version string to parse.
+
+    Returns:
+        A tuple of (version, optional_commit) where:
+        - version: The base version string (e.g., "1.0.0-dev0")
+        - optional_commit: The commit hash if present, None otherwise
+    """
+    # Check if this is a dev version with commit info
+    # Format: "1.0.0-dev0 (commit: 1234567)"
+    commit_pattern = r'^(.+) \(commit: ([a-f0-9]+)\)$'
+    match = re.match(commit_pattern, version)
+
+    if match:
+        base_version = match.group(1)
+        commit = match.group(2)
+        return base_version, commit
+    else:
+        # Regular version without commit info
+        return version, None
+
+
+def install_version_command(version: str, commit: Optional[str] = None) -> str:
+    if version == DEV_VERSION:
+        if commit is not None:
+            return ('pip install git+https://github.com/skypilot-org/skypilot@'
+                    f'{commit}')
+    elif 'dev' in version:
+        return f'pip install -U "skypilot-nightly=={version}"'
+    return f'pip install -U "skypilot=={version}"'
+
+
+def _remind_minor_version_upgrade(remote_version: str) -> None:
+    """Remind the user to upgrade the CLI/SDK."""
+    # Only print the reminder once per process.
+    global _reminded_for_minor_version_upgrade
+    if _reminded_for_minor_version_upgrade:
+        return
+    # Skip for dev versions.
+    if 'dev' in sky.__version__ or 'dev' in remote_version:
+        return
+
+    # Remove the commit info if any.
+    remote_base_version, _ = parse_readable_version(remote_version)
+
+    # Parse semver for both local and remote versions
+    try:
+        local = version_lib.parse(sky.__version__)
+        remote = version_lib.parse(remote_base_version)
+
+        # Check if local version is behind remote version, ignore patch version.
+        if (local.major, local.minor) < (remote.major, remote.minor):
+            logger.warning(
+                f'{colorama.Fore.YELLOW}The SkyPilot API server is running in '
+                f'version {remote_version}, which is newer than your client '
+                f'version {sky.__version__}. The compatibility for your '
+                f'current version might be dropped in the next server upgrade.'
+                f'\nConsider upgrading your client with:\n'
+                f'{install_version_command(remote_version)}'
+                f'{colorama.Style.RESET_ALL}')
+            _reminded_for_minor_version_upgrade = True
+    except version_lib.InvalidVersion:
+        # Skip for non-valid semver (probabely a dev version)
+        pass
+
+
+# TODO(aylei): maybe we can use similiar approach to mark a new argument can
+# only be used in the new server version.
+def minimal_api_version(min_version: int) -> Callable:
+    """Decorator to enforce a minimum remote API version for an SDK function.
+
+    New SDK method must be decorated with this decorator to make sure it raises
+    an readable error when the remote server is not upgraded.
+
+    Args:
+        min_version: The minimum remote API version required to call the
+        function.
+
+    Returns:
+        A decorator function that checks API version before execution.
+
+    Raises:
+        APINotSupportedError: If the remote API version is below the minimum
+            required.
+    """
+
+    def decorator(func: Callable) -> Callable:
+
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            remote_api_version = get_remote_api_version()
+            if remote_api_version is None:
+                return func(*args, **kwargs)
+            if remote_api_version < min_version:
+                with ux_utils.print_exception_no_traceback():
+                    hint = 'Please upgrade the remote server.'
+                    # The client runs in a released version, do better hint.
+                    if 'dev' not in sky.__version__:
+                        hint = (
+                            f'Upgrade the remote server to {sky.__version__} '
+                            'and re-run the command.')
+                    raise exceptions.APINotSupportedError(
+                        f'Function {func.__name__} is introduced after the '
+                        f'remote server version {get_remote_version()!r} is '
+                        f'released. {hint}')
+            return func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator

sky/setup_files/MANIFEST.in

@@ -17,4 +17,3 @@ include sky/utils/kubernetes/*
 include sky/server/html/*
 recursive-include sky/dashboard/out *
 include sky/users/*.conf
-include sky/metrics/*

{skypilot_nightly-1.0.0.dev20250702.dist-info → skypilot_nightly-1.0.0.dev20250704.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skypilot-nightly
-Version: 1.0.0.dev20250702
+Version: 1.0.0.dev20250704
 Summary: SkyPilot: Run AI on Any Infra — Unified, Faster, Cheaper.
 Author: SkyPilot Team
 License: Apache 2.0

{skypilot_nightly-1.0.0.dev20250702.dist-info → skypilot_nightly-1.0.0.dev20250704.dist-info}/RECORD

@@ -1,4 +1,4 @@
-sky/__init__.py,sha256=zvfkWi3ntfcQgmKXA4dcrMIKftYVQETwn9-qXK8JQqc,6419
+sky/__init__.py,sha256=8LPF74IsNl_AZzy1od36U0W_b93dBkCKW3e5LZWLLKs,6419
 sky/admin_policy.py,sha256=FMiizgvVTmD9gFA2OUaveXnuY3lbNU-fCbUYAODBZj4,9427
 sky/authentication.py,sha256=V7zGSV7bqcAKC_EGOOS0KhJ01ZFLnme0WnjLFO7zavs,25603
 sky/check.py,sha256=R0pFsTq2v-wr3NFePlX9DmDhsbvWEoFJAXsys3pUmT4,30338
@@ -6,7 +6,7 @@ sky/cli.py,sha256=VXIZryeTtJPYpPTBKymVPmuOCyh8knfWrq-qnkr6R-4,178
 sky/cloud_stores.py,sha256=Ln5GBpel-sEs7rVx7bBrMkfLwA_bctI05Rox2uoz7Lo,26388
 sky/core.py,sha256=5UbaZ5xamYTajEq4TyiT81Q_cVzqPnMQRXASSz4OcNg,56272
 sky/dag.py,sha256=5MFXlP43y9U54zxfYGhVyBiWEInoFFlt_zJ7ASJntXw,3889
-sky/exceptions.py,sha256=YRPgpFkBRGE9jmG2r3SWJGS9p9-RPP2TZGjfd2VblSc,19044
+sky/exceptions.py,sha256=Vn7VVhyrTaMBTdVxsro325aKSMM4_KRJOghFX31xIVM,19164
 sky/execution.py,sha256=BhI4paGKpXMhWJoelHFe7LGFbCnW7hl0rIAmdIay-dk,33223
 sky/global_user_state.py,sha256=8_oKM5l5aey_IadFC6beiilfsbG10J27sqVxnHxFc7A,73040
 sky/models.py,sha256=Eor-cT4D71QTimogcnJ5ey1G1PXK-OXN-snEtE8Uu_g,3152
@@ -34,7 +34,7 @@ sky/adaptors/vast.py,sha256=tpvmHi7IkQNzbbHVkeo04kUSajoEpSzXr2XgeO_I1LU,695
 sky/adaptors/vsphere.py,sha256=zJP9SeObEoLrpgHW2VHvZE48EhgVf8GfAEIwBeaDMfM,2129
 sky/backends/__init__.py,sha256=tpa9gAygQopsiBUUuy3wVmr4E05FoPTFHIWqEo4i-u0,627
 sky/backends/backend.py,sha256=o47WUnB_h2nd_SkV0q0NTJ4vCwk23-KH5DgAm_JpKgE,7739
-sky/backends/backend_utils.py,sha256=HFuEwwO28H-V3ytr18rNFMr1ol9sAroG8VeXTgKi9YQ,145088
+sky/backends/backend_utils.py,sha256=1FlFUQbJfwWBs8kaBP4oCH6K7frcLYTewivh3xqJG5I,145117
 sky/backends/cloud_vm_ray_backend.py,sha256=ILAOxfN4tZJolbmx7Sl1kTaSnd-sZdNTtvZKFbfzDoM,259464
 sky/backends/docker_utils.py,sha256=Hyw1YY20EyghhEbYx6O2FIMDcGkNzBzV9TM7LFynei8,8358
 sky/backends/local_docker_backend.py,sha256=r80BGJZmAH8F49v6Y_pG3_pHmW5LQEQRusLkKoYoe9Q,17047
@@ -65,7 +65,7 @@ sky/catalog/data_fetchers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJW
 sky/catalog/data_fetchers/analyze.py,sha256=ftnrOceREmjimFWBLwhtF2aqOnOZNl6TmYxqD6eIhHY,2069
 sky/catalog/data_fetchers/fetch_aws.py,sha256=RTkqG2LE2YxRoK4K4p_hq-vA9tsPr9UoWIiNamGyLyI,23406
 sky/catalog/data_fetchers/fetch_azure.py,sha256=7YVnoGDGGZI2TK02bj_LOoD4E5J5CFl6eqz2XlR4Vy8,12790
-sky/catalog/data_fetchers/fetch_cudo.py,sha256=52P48lvWN0s1ArjeLPeLemPRpxjSRcHincRle0nqdm4,3440
+sky/catalog/data_fetchers/fetch_cudo.py,sha256=hmVt7yTQ50EFPhum3rstKdy9cxv2GoxkpDaSRFP7ITc,3635
 sky/catalog/data_fetchers/fetch_fluidstack.py,sha256=hsqpQi_YUI-qil3zLCEGatrR7BkWzywr4otRdHrd-4k,7350
 sky/catalog/data_fetchers/fetch_gcp.py,sha256=y3boC2NBLRErLyY12078AS7QWGx9ItrzmFcO17h_VH8,31925
 sky/catalog/data_fetchers/fetch_hyperbolic.py,sha256=VLBJXWvrKJH5QbMUiREPEMfO3x37_XsqMAmOo7bCQtc,4537
@@ -76,17 +76,17 @@ sky/catalog/data_fetchers/fetch_vsphere.py,sha256=Yf7tKzwJsQ_4f64IT1EAP108C1D3Rg
 sky/client/__init__.py,sha256=pz6xvVSd9X-gwqbsDL0E9QOojYqM0KAD0j-NCyCIF1k,38
 sky/client/common.py,sha256=E_5cjxd8fWRB7fU1yfIbiyQf-IyVhpD5KkB7Fl3cQEI,15215
 sky/client/oauth.py,sha256=sNJ_DMsSTcxluj5FeNQ2IafZJLImRFmCAZ79bXeABn4,2871
-sky/client/sdk.py,sha256=uVzzJpsokxAdhLaK4JyMRia1wj6PI5qYpvBSZfnFwqk,91054
+sky/client/sdk.py,sha256=hZ2SLsHItKJOs-t_suVDMxIrvEixr1ZWzDehkOYycSc,91024
 sky/client/service_account_auth.py,sha256=5jXk0G6ufuW-SHCO7BEHQeTO0_2a8KfFmA63auXFRj4,1529
 sky/client/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/client/cli/command.py,sha256=tuK-39aOz9xresShPe4uvGAjn7OgnyxLovnycJUAzaE,217309
 sky/client/cli/deprecation_utils.py,sha256=H_d5UyF2CekEoThduAzt5cihBO8hwKYMu0-Wqfbjv5E,3370
 sky/client/cli/flags.py,sha256=6IdS_G2lB1C6r8kJMzLVREY_qV50VHiSfWlmppVrWuc,11623
 sky/clouds/__init__.py,sha256=tC9_Vi6GvVceWUi6uZvm7vXzBa0uH3CjRQ-QCYqdJMw,1624
-sky/clouds/aws.py,sha256=SDCYjS_4PsICbhnG9a1hf-ufOd8MBQ4ErK7ZKYjYIJM,55189
+sky/clouds/aws.py,sha256=A7ouOPS4vvh4rLbwMLTyAW5aih9d151Fz7KvHfo7b9I,55192
 sky/clouds/azure.py,sha256=Tm2kl-p736MGL9z-r7HTADneRKEVg-p2Lq3yHVaLAn8,32683
 sky/clouds/cloud.py,sha256=R4bdXwIWx66GkBeFFRdapZOQTTB7pSFonCRqmVT25_s,38995
-sky/clouds/cudo.py,sha256=DvKod2xbCqd7wCXzwC3ePtz9GRLZhCiBl09QJVLWwYg,13643
+sky/clouds/cudo.py,sha256=L657d8oD9XP8upQjWmG8h-MHmhS0C81FEhcwqdbZ6so,13648
 sky/clouds/do.py,sha256=x1oxaoGL3ioFJFwt3BA1N4seGyetPpV23ENemQLDe2I,11999
 sky/clouds/fluidstack.py,sha256=FYjTOYXulqUx_nH7iL3CpHTAxMMyFymKIW35LHtuvaM,13054
 sky/clouds/gcp.py,sha256=lXKymalvUeqS7XcBxZXXFVd1Z6r8TXtMTte7npTqEf8,68326
@@ -108,19 +108,19 @@ sky/clouds/utils/azure_utils.py,sha256=NToRBnhEyuUvb-nBnsKTxjhOBRkMcrelL8LK4w6s4
 sky/clouds/utils/gcp_utils.py,sha256=ZNupZEBsZS9ZvvzB7vsc6f5y1JUBoRcKE0erso_Y93k,7659
 sky/clouds/utils/oci_utils.py,sha256=yv_Y9oM5WNCnOofu44aKyOPTZZdKfpFLCx3ewZ2VBFY,7994
 sky/clouds/utils/scp_utils.py,sha256=VGuccVO5uFGr8-yolWSoYrgr11z6cIeDBGcqkBzAyOs,18409
-sky/dashboard/out/404.html,sha256=yR0gHfhgYY3Nqefhv2EDtn7lda5COkbUnQRPxvSe6Hc,1423
-sky/dashboard/out/clusters.html,sha256=Vq_CaBEdMFrpWSbWqYmV2EwY8g_iLuoYkhrzp-kI1Kk,1418
-sky/dashboard/out/config.html,sha256=Wv9UMBqjMGRJajReNMUGFbZrWKqVres0MaIfHl0DODI,1414
+sky/dashboard/out/404.html,sha256=yNzQQsN9o6oyq0e-av29_xLDT1r1OTML7o9TWGuS0m0,1423
+sky/dashboard/out/clusters.html,sha256=0rJ4h7PP3KGRHOrmAGPP7lvXuJVkLpIkiHRGxQZFY_4,1418
+sky/dashboard/out/config.html,sha256=rKr2CZvuJm1U2NFfNiEvNCka0yJB2d_V_JHcTVA2Iik,1414
 sky/dashboard/out/favicon.ico,sha256=XilUZZglAl_1zRsg85QsbQgmQAzGPQjcUIJ-A3AzYn8,93590
-sky/dashboard/out/index.html,sha256=bujiqQK9MkGC81TpTKRt8HclBSHsKVHBuApms6EgoBA,1407
-sky/dashboard/out/infra.html,sha256=x8KQCNDphnAehxphBuQdEawNUbqefSO_d7pJ-JdZCP8,1412
-sky/dashboard/out/jobs.html,sha256=_Y5_dDzlB7jt8M0B5y2U5gi2Ngx_O5ZAHKYmqvr-xCc,1410
+sky/dashboard/out/index.html,sha256=Euf9Dw7IjOxchHvRWdFygR8SkMsn0QpcVeC4iFG2kiI,1407
+sky/dashboard/out/infra.html,sha256=Ppikha45vtEboyLPM2frMRSlXN5JUQ9VZp_yiGYP1MQ,1412
+sky/dashboard/out/jobs.html,sha256=aziBpqaL9ghltymq6J8WslEaFg78f90tmdaAMLd5GL0,1410
 sky/dashboard/out/skypilot.svg,sha256=c0iRtlfLlaUm2p0rG9NFmo5FN0Qhf3pq5Xph-AeMPJw,5064
-sky/dashboard/out/users.html,sha256=qrUKhocHSDU1LRg4N25-EkuXY3rVWk_7uup5SCM1gbA,1412
-sky/dashboard/out/volumes.html,sha256=Ji2EKpMhKqwVq7Fjc8S9rubatOnowkdpgaetmHMh82E,1416
-sky/dashboard/out/workspaces.html,sha256=HR2tiOtj5PX3aIgJw8W3Ju2Gg2P8b9p_r86f1wwCRLw,1422
-sky/dashboard/out/_next/static/N5IdFnjR1RaPGBAVYeTIr/_buildManifest.js,sha256=SMKLljkiS_9UQoF0v1MyY_ziwbXl8KnBBlwCuTRc3V8,2235
-sky/dashboard/out/_next/static/N5IdFnjR1RaPGBAVYeTIr/_ssgManifest.js,sha256=Z49s4suAsf5y_GfnQSvm4qtq2ggxEbZPfEDTXjy6XgA,80
+sky/dashboard/out/users.html,sha256=QF18NuylmTUhFshF7ArVA6vE2pLFymlclX0D6CHgJJw,1412
+sky/dashboard/out/volumes.html,sha256=OOjj7wqNfoNem-U7_NzW-pmrZ5qj_71lx0bmG6z_f68,1416
+sky/dashboard/out/workspaces.html,sha256=VRpJVIhkZw5z8iLNZO9sJP3QbXWDwMhLDonxy5L4w3Q,1422
+sky/dashboard/out/_next/static/6TieQqyqsJiaJC33q0FfI/_buildManifest.js,sha256=SMKLljkiS_9UQoF0v1MyY_ziwbXl8KnBBlwCuTRc3V8,2235
+sky/dashboard/out/_next/static/6TieQqyqsJiaJC33q0FfI/_ssgManifest.js,sha256=Z49s4suAsf5y_GfnQSvm4qtq2ggxEbZPfEDTXjy6XgA,80
 sky/dashboard/out/_next/static/chunks/1043-1b39779691bb4030.js,sha256=k8-hHIedPKi22M0wZwGOdWZ_S-H0jVnepFwm7i9_zg4,18192
 sky/dashboard/out/_next/static/chunks/1141-726e5a3f00b67185.js,sha256=lEAVPk_18NZpZuPrGx1Xy9VWqE_5n3JuqWbIPS1_ZMw,17823
 sky/dashboard/out/_next/static/chunks/1272-1ef0bf0237faccdb.js,sha256=VJ6y-Z6Eg2T93hQIRfWAbjAkQ7nQhglmIaVbEpKSILY,38451
@@ -174,13 +174,13 @@ sky/dashboard/out/_next/static/chunks/pages/jobs/[job]-c4d5cfac7fbc0668.js,sha25
 sky/dashboard/out/_next/static/chunks/pages/workspace/new-5629d4e551dba1ee.js,sha256=K9tqKHcB2kiSHTAddLaM2oL1PzmqZNTdLDOuNzCaJNM,765
 sky/dashboard/out/_next/static/chunks/pages/workspaces/[name]-7c0187f43757a548.js,sha256=FsoenC6VRkPLE29KkLd4nlSq2qociH4zFexYMxgetOM,1530
 sky/dashboard/out/_next/static/css/0da6afe66176678a.css,sha256=I8b7x4ULdoXHkgk0BsNa0D6k2UYoA2mEhq7H1G_tVQE,44869
-sky/dashboard/out/clusters/[cluster].html,sha256=qYiMQO245xkWrOAYe11EfaLrWj84EbxuSdiZe8Iua0w,2847
-sky/dashboard/out/clusters/[cluster]/[job].html,sha256=8_YZlQl4GTSVr0CNJAZo4KAHptEmECn4uPx-g04_nEU,2160
-sky/dashboard/out/infra/[context].html,sha256=3v2yCWMr3JQ3d-tTHh6Etia72wZTniMtPkzvG-S0xGI,1436
-sky/dashboard/out/jobs/[job].html,sha256=lhlOnKyLEBpx6URDwI24qtk4xUwUsapdX_OXextT5Aw,2304
+sky/dashboard/out/clusters/[cluster].html,sha256=lxJjYd3ccjhdhqCoJbdqxqQJb-J9nW7hOhGHg1Cad0w,2847
+sky/dashboard/out/clusters/[cluster]/[job].html,sha256=3dpg96pStIpsO21nEAh0mQvjCrHmrsOmoq7u1xXf1Vo,2160
+sky/dashboard/out/infra/[context].html,sha256=zB4Y-_FgAUCUZ-_TUvxXxNqdXjDbvJmWNZT9F-OlaWI,1436
+sky/dashboard/out/jobs/[job].html,sha256=ZBj9RP-fE-lNYtoqg2LK4BJua_0zlOhq0Fw-WmqFFqM,2304
 sky/dashboard/out/videos/cursor-small.mp4,sha256=8tRdp1vjawOrXUar1cfjOc-nkaKmcwCPZx_LO0XlCvQ,203285
-sky/dashboard/out/workspace/new.html,sha256=iauAVpCteFD0PZ4jV8M44Qj_zL5VUu6n3BSHcQ60kT0,1428
-sky/dashboard/out/workspaces/[name].html,sha256=NEluCe56SZiw_t6lbftdgqVNNsAqGfWpi5SA5x_USXg,2845
+sky/dashboard/out/workspace/new.html,sha256=XSCLSV2bZqtbyD-eqS7mRyLOhLBpucznrYXxdx2tEyI,1428
+sky/dashboard/out/workspaces/[name].html,sha256=M0x3-_wm2L2VSRSY-0SzCHkSpMjZLbuTUw1Ln3d1kK0,2845
 sky/data/__init__.py,sha256=Nhaf1NURisXpZuwWANa2IuCyppIuc720FRwqSE2oEwY,184
 sky/data/data_transfer.py,sha256=N8b0CQebDuHieXjvEVwlYmK6DbQxUGG1RQJEyTbh3dU,12040
 sky/data/data_utils.py,sha256=CNYPM963qby5ddW0DZNbhiWXkqgB9MHh_jrC5DoBctM,33437
@@ -195,13 +195,14 @@ sky/jobs/scheduler.py,sha256=b3RAjEzCXyoikh_BcmmGjoZ9ZeXr-tBnXoLFctt95ko,14375
 sky/jobs/state.py,sha256=HHERdEhuJ3QHMu5dvY2mHVJgtHzIkAI6F1Cv2Iz3Hvw,65516
 sky/jobs/utils.py,sha256=gfW4smCWWMmE4OwIbEqI5j5gVh5Gvfs6paqKQR2Tia8,70920
 sky/jobs/client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sky/jobs/client/sdk.py,sha256=0ZDm5h1kqPYXvsgiXBhbsXj9jfKW7GHH_VtQbSQjmCw,11068
+sky/jobs/client/sdk.py,sha256=Nw7bP8pJ-uLIi14kMekEtmzqPmdIiN-CbhoNCJlHA7s,11340
 sky/jobs/server/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-sky/jobs/server/core.py,sha256=SioPzD6O0OO-_mg0vA-ND3d_4SPsMs5W9eStMtwfp_U,31405
+sky/jobs/server/core.py,sha256=U9vkOAcg381yh_te-ZTXxDgQ1CrxAVzOx9dUkzXuELo,31992
 sky/jobs/server/server.py,sha256=ae8JAs-2ipWqL_GsA3x8T2mY-OJLc3ioWg_CfRzCUIY,4011
 sky/logs/__init__.py,sha256=0ybWMfXcpAzh8dtDnJwpfovNIk0zJRJvzdISqFdKmdE,549
 sky/logs/agent.py,sha256=tv0C40_FauZpvU93Ro_mC23LnaXWhSTjqch1JQMXiqw,2771
 sky/logs/gcp.py,sha256=eKVEcHO3FJRg_YTcE9omE8uAIF6AdBWWpkEPqWFAqXg,3857
+sky/metrics/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/metrics/utils.py,sha256=Cww3yNG4HyW4DEdLOFUayFgMZ16t2JFSvvhuTTV7Vio,7654
 sky/provision/__init__.py,sha256=bADZkpZdy7QxIc7WDkh-X5cd3Uv64wy2K7IB9Xpw348,7554
 sky/provision/common.py,sha256=LdjM9SL9NDtsARom12tVv_WoUNL3PTlU5RoLfeWGGgM,10807
@@ -222,8 +223,8 @@ sky/provision/azure/instance.py,sha256=BQ4stZl_m76iULPY_TFEN4WDwNPWI1jmw-LswxxqX
 sky/provision/cudo/__init__.py,sha256=KAEl26MVPsk7IoP9Gg-MOJJRIV6-X9B0fbyHdyJWdLo,741
 sky/provision/cudo/config.py,sha256=RYOVkV0MoUqVBJRZiKhBZhjFygeyFs7eUdVMdPg1vds,327
 sky/provision/cudo/cudo_machine_type.py,sha256=Bo2z4DyIvCFe98kYD8itTADx6MG7tn8ny8Jbh6QYbGg,681
-sky/provision/cudo/cudo_utils.py,sha256=fIwRpCSOXOyIcCLh-kGMdfl-ACTXdTwyHf6o27rMA6Y,1730
-sky/provision/cudo/cudo_wrapper.py,sha256=WgtA1-ZMytR7tG7UFOH93hSvSAqDWymwWxQREspbZAQ,4678
+sky/provision/cudo/cudo_utils.py,sha256=GFwSEf_LXN7Dh37m5UOLs6wXEJqF8kGI2-ys35L4rl8,1883
+sky/provision/cudo/cudo_wrapper.py,sha256=JG9s8_qh8IR8E-D_x4Mokyv7ZJzSky38sbcD3pnyzss,4919
 sky/provision/cudo/instance.py,sha256=C2tl48PtQWz0irDWiSNoqWu8YcrDqQrucau0KGUFFxg,8828
 sky/provision/do/__init__.py,sha256=VAhSO9F9FDAjvEuCDlRPueB-TTA3XHR61vVXyUe85z0,518
 sky/provision/do/config.py,sha256=WWFz6kZLKrUHoJoKRzHGY8ck_ieQiBrMWDb5JP6rWUk,387
@@ -320,31 +321,32 @@ sky/serve/server/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
 sky/serve/server/core.py,sha256=zKvDnq6W9CftH5ljmNMdTDGsLO3mzbFgrMBMCSua4dM,41662
 sky/serve/server/server.py,sha256=A9K37a0nQgZeN3eKWv62Oh2C5TSAReTZ9pHmztqlI-c,4396
 sky/server/__init__.py,sha256=MPPBqFzXz6Jv5QSk6td_IcvnfXfNErDZVcizu4MLRow,27
-sky/server/common.py,sha256=LKZPf-jaNxIMCG_bFetFcq3g8xB4MSOBuUzjEB1xQEI,36028
+sky/server/common.py,sha256=u7hV5oGdCj5GfCxpNmLd6yl0B1mlh11FMFIR5ZoW5pA,34492
 sky/server/config.py,sha256=XWf5Kw4am6vMO5wcyWevbQAFH-dmKb7AMEgDzD083-M,8538
-sky/server/constants.py,sha256=15r9CGX4yo62BeRLSfyrVaKd5fqeu_8GBS_JqyeXSfk,1431
+sky/server/constants.py,sha256=v6RP8ZEwCNsxO2s2IukgOcEldZO5bvcQaxmOTA0Cf0g,2321
 sky/server/metrics.py,sha256=aVRaSwpBVXE9dXIVd9bNsSigKM4bkqNq0eTpP0Noyo8,3657
-sky/server/rest.py,sha256=3xOQXsQ_r9XBcUOhQbf-Wk0UXx0XrAmzQ6JSqLr6nJ0,5430
-sky/server/server.py,sha256=OOg8ZvF4NAbo-qwwD0B6ZlUibcR2kaHSrTsoCoRrTp8,70703
+sky/server/rest.py,sha256=elMpi-cS301OqTwmFZb7fWYXr1vdJ28YT1zIDH1IpBk,5968
+sky/server/server.py,sha256=Zv9WgfGfg9dnGQaEfA5mxsua9x3vp067TvCiCxHDIKA,72833
 sky/server/state.py,sha256=YbVOMJ1JipQQv17gLIGyiGN7MKfnP83qlUa5MB1z0Yk,747
 sky/server/stream_utils.py,sha256=RS4RuMxQqTGqp3uxzZVtmFWzos4d49P7hMX_VklzEVU,9189
 sky/server/uvicorn.py,sha256=3mdSUbc8zHRYAbZZLkfPB6U9VXD_t2jDM1BMjpTx1Mo,9014
+sky/server/versions.py,sha256=3atZzUa7y1XeKNcrfVxKWAo_5ZyCOnbY7DKpIqed7Do,10011
 sky/server/html/log.html,sha256=TSGZktua9Ysl_ysg3w60rjxAxhH61AJnsYDHdtqrjmI,6929
 sky/server/html/token_page.html,sha256=eUndS5u1foL9vaWGPRTLMt7lCzD1g0wYJ2v_EeeFzlc,7046
 sky/server/requests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/server/requests/event_loop.py,sha256=OhpPbuce65bbjpGRlcJa78AVnYSm08SzFKt70ypCUuQ,1211
 sky/server/requests/executor.py,sha256=v_ae2H9GGMiwxbVT9h0J2fUOSjqIs3F9lKbMtrzb1-k,26476
-sky/server/requests/payloads.py,sha256=12UdUnAprNc8DmhbeLjL-HhAxXPGTsk_Qzd3iDzAdBY,21498
+sky/server/requests/payloads.py,sha256=P4Qv8fWQD6Xz7hP-dMvsA63SpPbWjYespgO5-ch2wik,23334
 sky/server/requests/preconditions.py,sha256=uUQjzFFHf7O5-WvBypMzqViGmd1CXksbqrrDPmY_s_Y,7178
 sky/server/requests/process.py,sha256=I0ToOcF_cMJi5TfZMJOn9rDmJYewQOV07Pnetzqj2IU,10805
-sky/server/requests/requests.py,sha256=bz2TTgynYz1ip93SBf4_vFLVHc-WtuNpQ1Tnd3gQHZs,25625
+sky/server/requests/requests.py,sha256=AT4nJKCa2I13rVhZq929hMtZF5-So7w-gp9E_Ld60Xk,25193
 sky/server/requests/queues/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/server/requests/queues/local_queue.py,sha256=X6VkBiUmgd_kfqIK1hCtMWG1b8GiZbY70TBiBR6c6GY,416
 sky/server/requests/queues/mp_queue.py,sha256=jDqP4Jd28U3ibSFyMR1DF9I2OWZrPZqFJrG5S6RFpyw,3403
 sky/server/requests/serializers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/server/requests/serializers/decoders.py,sha256=qphN79pRAaaitCbcsZIrslphgZn1iYndl6JnmergEe4,6361
 sky/server/requests/serializers/encoders.py,sha256=4bQV5yTg8RTPT_HkRyQpjaBY_uUvBJ4NH189W0-6Pi0,5578
-sky/setup_files/MANIFEST.in,sha256=9osXc1PANaHXqD1MVrZIf-koWCiU4GUUL7NBe-0y5oY,666
+sky/setup_files/MANIFEST.in,sha256=BzWGsYZz9fO40HN1Uxm1Ca4pn1FkKn3lcoY1B9-YRiA,644
 sky/setup_files/dependencies.py,sha256=OY4-KzEp2MObl4vTUUFD_UVXXb4IYFl51_O4iZny8Jg,7127
 sky/setup_files/setup.py,sha256=GTXvAi65S4_TSLhQ1GzkmaWf_yzciHiaxMbZumcTtKU,7522
 sky/skylet/LICENSE,sha256=BnFrJSvUFpMUoH5mOpWnEvaC5R6Uux8W6WXgrte8iYg,12381
@@ -477,9 +479,9 @@ sky/workspaces/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/workspaces/core.py,sha256=MkQoVqWN67tf4VRq284U9vgAw4lwb_cpUfwHQT4V9Ow,16598
 sky/workspaces/server.py,sha256=Box45DS54xXGHy7I3tGKGy-JP0a8G_z6IhfvGlEXtsA,3439
 sky/workspaces/utils.py,sha256=IIAiFoS6sdb2t0X5YoX9AietpTanZUQNTK8cePun-sY,2143
-skypilot_nightly-1.0.0.dev20250702.dist-info/licenses/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
-skypilot_nightly-1.0.0.dev20250702.dist-info/METADATA,sha256=a-sgt0ERBnbGvrYW3np8kpboo3h4MD6ygJxyhxA1HYk,18908
-skypilot_nightly-1.0.0.dev20250702.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-skypilot_nightly-1.0.0.dev20250702.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
-skypilot_nightly-1.0.0.dev20250702.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
-skypilot_nightly-1.0.0.dev20250702.dist-info/RECORD,,
+skypilot_nightly-1.0.0.dev20250704.dist-info/licenses/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
+skypilot_nightly-1.0.0.dev20250704.dist-info/METADATA,sha256=IhXJKqXPO1QKORMRMFI0hQidarSR6N7yvDbv43VHvFI,18908
+skypilot_nightly-1.0.0.dev20250704.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+skypilot_nightly-1.0.0.dev20250704.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
+skypilot_nightly-1.0.0.dev20250704.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
+skypilot_nightly-1.0.0.dev20250704.dist-info/RECORD,,