skypilot-nightly 1.0.0.dev20250627__py3-none-any.whl → 1.0.0.dev20250630__py3-none-any.whl

sky/provision/kubernetes/volume.py

@@ -1,10 +1,12 @@
 """Kubernetes pvc provisioning."""
 from typing import Any, Dict, List, Optional, Tuple
 
+from sky import global_user_state
 from sky import models
 from sky import sky_logging
 from sky.adaptors import kubernetes
 from sky.provision.kubernetes import config as config_lib
+from sky.provision.kubernetes import constants as k8s_constants
 from sky.provision.kubernetes import utils as kubernetes_utils
 from sky.volumes import volume as volume_lib
 
@@ -45,17 +47,26 @@ def check_pvc_usage_for_pod(context: Optional[str], namespace: str,
         access_mode = pvc.spec.access_modes[0]
         if access_mode not in once_modes:
             continue
-        usedby = _get_volume_usedby(context, namespace, pvc_name)
-        if usedby:
+        usedby_pods, _ = _get_volume_usedby(context, namespace, pvc_name)
+        if usedby_pods:
             raise config_lib.KubernetesError(f'Volume {pvc_name} with access '
                                              f'mode {access_mode} is already '
-                                             f'in use by {usedby}.')
+                                             f'in use by Pods {usedby_pods}.')
 
 
 def apply_volume(config: models.VolumeConfig) -> models.VolumeConfig:
     """Creates or registers a volume."""
     context, namespace = _get_context_namespace(config)
     pvc_spec = _get_pvc_spec(namespace, config)
+    # Check if the storage class exists
+    storage_class_name = pvc_spec['spec'].get('storageClassName')
+    if storage_class_name is not None:
+        try:
+            kubernetes.storage_api(context).read_storage_class(
+                name=storage_class_name)
+        except kubernetes.api_exception() as e:
+            raise config_lib.KubernetesError(
+                f'Check storage class {storage_class_name} error: {e}')
     create_persistent_volume_claim(namespace, context, pvc_spec)
     return config
 
@@ -76,22 +87,73 @@ def delete_volume(config: models.VolumeConfig) -> models.VolumeConfig:
     return config
 
 
-def _get_volume_usedby(context: Optional[str], namespace: str,
-                       pvc_name: str) -> List[str]:
-    """Gets the usedby resources of a volume."""
-    usedby = []
+def _get_volume_usedby(
+    context: Optional[str],
+    namespace: str,
+    pvc_name: str,
+) -> Tuple[List[str], List[str]]:
+    """Gets the usedby resources of a volume.
+
+    This function returns the pods and clusters that are using the volume.
+    The usedby_pods is accurate, which also includes the Pods that are not
+    managed by SkyPilot.
+
+    Args:
+        context: Kubernetes context
+        namespace: Kubernetes namespace
+        pvc_name: PVC name
+
+    Returns:
+        usedby_pods: List of pods using the volume. These may include pods
+                     not created by SkyPilot.
+        usedby_clusters: List of clusters using the volume.
+    """
+    usedby_pods = []
+    usedby_clusters = []
+    field_selector = ','.join([
+        f'status.phase!={phase}'
+        for phase in k8s_constants.PVC_NOT_HOLD_POD_PHASES
+    ])
+    cloud_to_name_map = _get_cluster_name_on_cloud_to_cluster_name_map()
     # Get all pods in the namespace
-    pods = kubernetes.core_api(context).list_namespaced_pod(namespace=namespace)
+    pods = kubernetes.core_api(context).list_namespaced_pod(
+        namespace=namespace, field_selector=field_selector)
     for pod in pods.items:
-        if pod.spec.volumes is not None:
-            for volume in pod.spec.volumes:
-                if volume.persistent_volume_claim is not None:
-                    if volume.persistent_volume_claim.claim_name == pvc_name:
-                        usedby.append(pod.metadata.name)
-    return usedby
+        if pod.spec.volumes is None:
+            continue
+        for volume in pod.spec.volumes:
+            if volume.persistent_volume_claim is None:
+                continue
+            if volume.persistent_volume_claim.claim_name == pvc_name:
+                usedby_pods.append(pod.metadata.name)
+                # Get the real cluster name
+                cluster_name_on_cloud = pod.metadata.labels.get(
+                    k8s_constants.TAG_SKYPILOT_CLUSTER_NAME)
+                if cluster_name_on_cloud is None:
+                    continue
+                cluster_name = cloud_to_name_map.get(cluster_name_on_cloud)
+                if cluster_name is not None:
+                    usedby_clusters.append(cluster_name)
+    if usedby_pods:
+        logger.debug(f'Volume {pvc_name} is used by Pods {usedby_pods}'
+                     f' and clusters {usedby_clusters}')
+    return usedby_pods, usedby_clusters
+
+
+def _get_cluster_name_on_cloud_to_cluster_name_map() -> Dict[str, str]:
+    """Gets the map from cluster name on cloud to cluster name."""
+    clusters = global_user_state.get_clusters()
+    cloud_to_name_map = {}
+    for cluster in clusters:
+        handle = cluster['handle']
+        if handle is None:
+            continue
+        cloud_to_name_map[handle.cluster_name_on_cloud] = cluster['name']
+    return cloud_to_name_map
 
 
-def get_volume_usedby(config: models.VolumeConfig) -> List[str]:
+def get_volume_usedby(
+    config: models.VolumeConfig,) -> Tuple[List[str], List[str]]:
     """Gets the usedby resources of a volume."""
     context, namespace = _get_context_namespace(config)
     pvc_name = config.name_on_cloud

sky/provision/nebius/utils.py

@@ -40,8 +40,11 @@ def get_project_by_region(region: str) -> str:
         parent_id=nebius.get_tenant_id())).wait()
 
     #  Check is there project if in config
-    project_id = skypilot_config.get_nested(('nebius', region, 'project_id'),
-                                            None)
+    project_id = skypilot_config.get_effective_region_config(
+        cloud='nebius',
+        region=None,
+        keys=(region, 'project_id'),
+        default_value=None)
     if project_id is not None:
         return project_id
     for project in projects.items:
@@ -184,8 +187,11 @@ def launch(cluster_name_on_cloud: str,
     # https://docs.nebius.com/compute/clusters/gpu
     if platform in nebius_constants.INFINIBAND_INSTANCE_PLATFORMS:
         if preset == '8gpu-128vcpu-1600gb':
-            fabric = skypilot_config.get_nested(('nebius', region, 'fabric'),
-                                                None)
+            fabric = skypilot_config.get_effective_region_config(
+                cloud='nebius',
+                region=None,
+                keys=(region, 'fabric'),
+                default_value=None)
 
             # Auto-select fabric if network_tier=best and no fabric configured
             if (fabric is None and

sky/resources.py

@@ -1064,8 +1064,11 @@ class Resources:
             regions = [r for r in regions if r.name in self._image_id]
 
         # Filter the regions by the skypilot_config
-        ssh_proxy_command_config = skypilot_config.get_nested(
-            (str(self._cloud).lower(), 'ssh_proxy_command'), None)
+        ssh_proxy_command_config = skypilot_config.get_effective_region_config(
+            cloud=str(self._cloud).lower(),
+            region=None,
+            keys=('ssh_proxy_command',),
+            default_value=None)
         if (isinstance(ssh_proxy_command_config, str) or
                 ssh_proxy_command_config is None):
             # All regions are valid as the regions are not specified for the
@@ -1550,8 +1553,11 @@ class Resources:
             # to each cloud if any cloud supports reservations for spot.
             return {}
         specific_reservations = set(
-            skypilot_config.get_nested(
-                (str(self.cloud).lower(), 'specific_reservations'), set()))
+            skypilot_config.get_effective_region_config(
+                cloud=str(self.cloud).lower(),
+                region=self.region,
+                keys=('specific_reservations',),
+                default_value=set()))
 
         if isinstance(self.cloud, clouds.DummyCloud):
             return self.cloud.get_reservations_available_resources(

sky/serve/client/sdk.py

@@ -74,12 +74,11 @@ def up(
             task=dag_str,
             service_name=service_name,
         )
-        response = rest.post(
-            f'{server_common.get_server_url()}/serve/up',
+        response = server_common.make_authenticated_request(
+            'POST',
+            '/serve/up',
             json=json.loads(body.model_dump_json()),
-            timeout=(5, None),
-            cookies=server_common.get_api_cookie_jar(),
-        )
+            timeout=(5, None))
         return server_common.get_request_id(response)
 
 
@@ -136,12 +135,11 @@ def update(
             mode=mode,
         )
 
-        response = rest.post(
-            f'{server_common.get_server_url()}/serve/update',
+        response = server_common.make_authenticated_request(
+            'POST',
+            '/serve/update',
             json=json.loads(body.model_dump_json()),
-            timeout=(5, None),
-            cookies=server_common.get_api_cookie_jar(),
-        )
+            timeout=(5, None))
         return server_common.get_request_id(response)
 
 
@@ -178,12 +176,11 @@ def down(
         all=all,
         purge=purge,
     )
-    response = rest.post(
-        f'{server_common.get_server_url()}/serve/down',
+    response = server_common.make_authenticated_request(
+        'POST',
+        '/serve/down',
         json=json.loads(body.model_dump_json()),
-        timeout=(5, None),
-        cookies=server_common.get_api_cookie_jar(),
-    )
+        timeout=(5, None))
     return server_common.get_request_id(response)
 
 
@@ -213,12 +210,11 @@ def terminate_replica(service_name: str, replica_id: int,
         replica_id=replica_id,
         purge=purge,
     )
-    response = rest.post(
-        f'{server_common.get_server_url()}/serve/terminate-replica',
+    response = server_common.make_authenticated_request(
+        'POST',
+        '/serve/terminate-replica',
         json=json.loads(body.model_dump_json()),
-        timeout=(5, None),
-        cookies=server_common.get_api_cookie_jar(),
-    )
+        timeout=(5, None))
     return server_common.get_request_id(response)
 
 
@@ -286,12 +282,11 @@ def status(
         exceptions.ClusterNotUpError: if the sky serve controller is not up.
     """
     body = payloads.ServeStatusBody(service_names=service_names,)
-    response = rest.post(
-        f'{server_common.get_server_url()}/serve/status',
+    response = server_common.make_authenticated_request(
+        'POST',
+        '/serve/status',
         json=json.loads(body.model_dump_json()),
-        timeout=(5, None),
-        cookies=server_common.get_api_cookie_jar(),
-    )
+        timeout=(5, None))
     return server_common.get_request_id(response)
 
 
@@ -373,13 +368,12 @@ def tail_logs(service_name: str,
         replica_id=replica_id,
         follow=follow,
     )
-    response = rest.post(
-        f'{server_common.get_server_url()}/serve/logs',
+    response = server_common.make_authenticated_request(
+        'POST',
+        '/serve/logs',
         json=json.loads(body.model_dump_json()),
         timeout=(5, None),
-        stream=True,
-        cookies=server_common.get_api_cookie_jar(),
-    )
+        stream=True)
     request_id = server_common.get_request_id(response)
     return sdk.stream_response(request_id=request_id,
                                response=response,
@@ -436,11 +430,11 @@ def sync_down_logs(service_name: str,
         targets=targets,
         replica_ids=replica_ids,
     )
-    response = rest.post(
-        f'{server_common.get_server_url()}/serve/sync-down-logs',
+    response = server_common.make_authenticated_request(
+        'POST',
+        '/serve/sync-down-logs',
         json=json.loads(body.model_dump_json()),
-        timeout=(5, None),
-    )
+        timeout=(5, None))
     remote_dir = sdk.stream_and_get(server_common.get_request_id(response))
 
     # Download from API server paths to the client's local_dir

sky/server/common.py

@@ -27,6 +27,7 @@ from sky import exceptions
 from sky import sky_logging
 from sky import skypilot_config
 from sky.adaptors import common as adaptors_common
+from sky.client import service_account_auth
 from sky.data import data_utils
 from sky.server import constants as server_constants
 from sky.server import rest
@@ -185,6 +186,53 @@ def get_cookies_from_response(
     return cookies
 
 
+def make_authenticated_request(method: str,
+                               path: str,
+                               server_url: Optional[str] = None,
+                               retry: bool = True,
+                               **kwargs) -> 'requests.Response':
+    """Make an authenticated HTTP request to the API server.
+
+    Automatically handles service account token authentication or cookie-based
+    authentication based on what's available.
+
+    Args:
+        method: HTTP method (GET, POST, etc.)
+        path: API path (e.g., '/api/v1/status')
+        server_url: Server URL, defaults to configured server
+        **kwargs: Additional arguments to pass to requests
+
+    Returns:
+        requests.Response object
+    """
+    if server_url is None:
+        server_url = get_server_url()
+
+    # Prepare headers and URL for service account authentication
+    headers = service_account_auth.get_service_account_headers()
+
+    # Merge with existing headers
+    if 'headers' in kwargs:
+        headers.update(kwargs['headers'])
+    kwargs['headers'] = headers
+
+    # Always use the same URL regardless of authentication type
+    # OAuth2 proxy will handle authentication based on headers
+    url = f'{server_url}/{path}' if not path.startswith(
+        '/') else f'{server_url}{path}'
+
+    # Use cookie authentication if no Bearer token present
+    if not headers.get('Authorization') and 'cookies' not in kwargs:
+        kwargs['cookies'] = get_api_cookie_jar()
+
+    # Make the request
+    if retry:
+        return rest.request(method, url, **kwargs)
+    else:
+        assert method == 'GET', 'Only GET requests can be done without retry'
+        return rest.request_without_retry(method, url, **kwargs)
+
+
 @annotations.lru_cache(scope='global')
 def get_server_url(host: Optional[str] = None) -> str:
     endpoint = DEFAULT_SERVER_URL
@@ -243,9 +291,9 @@ def get_api_server_status(endpoint: Optional[str] = None) -> ApiServerInfo:
     server_url = endpoint if endpoint is not None else get_server_url()
     while time_out_try_count <= RETRY_COUNT_ON_TIMEOUT:
         try:
-            response = rest.get(f'{server_url}/api/health',
-                                timeout=2.5,
-                                cookies=get_api_cookie_jar())
+            response = make_authenticated_request('GET',
+                                                  '/api/health',
+                                                  timeout=2.5)
         except requests.exceptions.Timeout:
             if time_out_try_count == RETRY_COUNT_ON_TIMEOUT:
                 return ApiServerInfo(status=ApiServerStatus.UNHEALTHY)

sky/server/constants.py

@@ -36,3 +36,6 @@ API_COOKIE_FILE_DEFAULT_LOCATION = '~/.sky/cookies.txt'
 # The path to the dashboard build output
 DASHBOARD_DIR = os.path.join(os.path.dirname(__file__), '..', 'dashboard',
                              'out')
+
+# The interval (seconds) for the event to be restarted in the background.
+DAEMON_RESTART_INTERVAL_SECONDS = 20

sky/server/requests/executor.py

@@ -268,6 +268,10 @@ def override_request_env_and_config(
     user = models.User(id=request_body.env_vars[constants.USER_ID_ENV_VAR],
                        name=request_body.env_vars[constants.USER_ENV_VAR])
     global_user_state.add_or_update_user(user)
+    # Refetch the user to get the latest user info, including the created_at
+    # field.
+    user = global_user_state.get_user(user.id)
+
     # Force color to be enabled.
     os.environ['CLICOLOR_FORCE'] = '1'
     server_common.reload_for_new_request(

sky/server/requests/payloads.py

@@ -358,6 +358,39 @@ class UserImportBody(RequestBody):
     csv_content: str
 
 
+class ServiceAccountTokenCreateBody(RequestBody):
+    """The request body for creating a service account token."""
+    token_name: str
+    expires_in_days: Optional[int] = None
+
+
+class ServiceAccountTokenDeleteBody(RequestBody):
+    """The request body for deleting a service account token."""
+    token_id: str
+
+
+class UpdateRoleBody(RequestBody):
+    """The request body for updating a user role."""
+    role: str
+
+
+class ServiceAccountTokenRoleBody(RequestBody):
+    """The request body for getting a service account token role."""
+    token_id: str
+
+
+class ServiceAccountTokenUpdateRoleBody(RequestBody):
+    """The request body for updating a service account token role."""
+    token_id: str
+    role: str
+
+
+class ServiceAccountTokenRotateBody(RequestBody):
+    """The request body for rotating a service account token."""
+    token_id: str
+    expires_in_days: Optional[int] = None
+
+
 class DownloadBody(RequestBody):
     """The request body for the download endpoint."""
     folder_paths: List[str]

sky/server/requests/requests.py

@@ -375,10 +375,29 @@ def managed_job_status_refresh_event():
 
 @dataclasses.dataclass
 class InternalRequestDaemon:
+    """Internal daemon that runs an event in the background."""
+
     id: str
     name: str
     event_fn: Callable[[], None]
 
+    def run_event(self):
+        """Run the event."""
+        while True:
+            with ux_utils.enable_traceback():
+                try:
+                    self.event_fn()
+                    break
+                except Exception:  # pylint: disable=broad-except
+                    # It is OK to fail to run the event, as the event is not
+                    # critical, but we should log the error.
+                    logger.exception(
+                        f'Error running {self.name} event. '
+                        f'Restarting in '
+                        f'{server_constants.DAEMON_RESTART_INTERVAL_SECONDS} '
+                        'seconds...')
+                    time.sleep(server_constants.DAEMON_RESTART_INTERVAL_SECONDS)
+
 
 # Register the events to run in the background.
 INTERNAL_REQUEST_DAEMONS = [

sky/server/rest.py

@@ -129,25 +129,16 @@ def handle_server_unavailable(response: 'requests.Response') -> None:
 
 
 @retry_on_server_unavailable()
-def post(url, data=None, json=None, **kwargs) -> 'requests.Response':
-    """Send a POST request to the API server, retry on server temporarily
+def request(method, url, **kwargs) -> 'requests.Response':
+    """Send a request to the API server, retry on server temporarily
     unavailable."""
-    response = requests.post(url, data=data, json=json, **kwargs)
+    response = requests.request(method, url, **kwargs)
     handle_server_unavailable(response)
     return response
 
 
-@retry_on_server_unavailable()
-def get(url, params=None, **kwargs) -> 'requests.Response':
-    """Send a GET request to the API server, retry on server temporarily
-    unavailable."""
-    response = requests.get(url, params=params, **kwargs)
-    handle_server_unavailable(response)
-    return response
-
-
-def get_without_retry(url, params=None, **kwargs) -> 'requests.Response':
-    """Send a GET request to the API server without retry."""
-    response = requests.get(url, params=params, **kwargs)
+def request_without_retry(method, url, **kwargs) -> 'requests.Response':
+    """Send a request to the API server without retry."""
+    response = requests.request(method, url, **kwargs)
     handle_server_unavailable(response)
     return response

sky/server/server.py

@@ -119,8 +119,11 @@ def _basic_auth_401_response(content: str):
 # TODO(hailong): Remove this function and use request.state.auth_user instead.
 async def _override_user_info_in_request_body(request: fastapi.Request,
                                               auth_user: Optional[models.User]):
+    if auth_user is None:
+        return
+
     body = await request.body()
-    if auth_user and body:
+    if body:
         try:
             original_json = await request.json()
         except json.JSONDecodeError as e:
@@ -228,14 +231,17 @@ class BasicAuthMiddleware(starlette.middleware.base.BaseHTTPMiddleware):
 
     async def dispatch(self, request: fastapi.Request, call_next):
         if request.url.path.startswith('/api/'):
-            # Try to set the auth user from the basic auth header so the
-            # following endpoint handlers can leverage the auth_user info
+            # Try to set the auth user from basic auth
             _try_set_basic_auth_user(request)
             return await call_next(request)
 
         auth_header = request.headers.get('authorization')
-        if not auth_header or not auth_header.lower().startswith('basic '):
-            return _basic_auth_401_response('Invalid basic auth')
+        if not auth_header:
+            return _basic_auth_401_response('Authentication required')
+
+        # Only handle basic auth
+        if not auth_header.lower().startswith('basic '):
+            return _basic_auth_401_response('Invalid authentication method')
 
         # Check username and password
         encoded = auth_header.split(' ', 1)[1]
@@ -267,6 +273,111 @@ class BasicAuthMiddleware(starlette.middleware.base.BaseHTTPMiddleware):
         return await call_next(request)
 
 
+class BearerTokenMiddleware(starlette.middleware.base.BaseHTTPMiddleware):
+    """Middleware to handle Bearer Token Auth (Service Accounts)."""
+
+    async def dispatch(self, request: fastapi.Request, call_next):
+        # Only process requests with Bearer token authorization header
+        auth_header = request.headers.get('authorization')
+        if not auth_header or not auth_header.lower().startswith('bearer '):
+            # No Bearer token, continue with normal processing (OAuth2 cookies,
+            # etc.)
+            return await call_next(request)
+
+        # Extract token
+        sa_token = auth_header.split(' ', 1)[1]
+
+        # Handle SkyPilot service account tokens
+        if sa_token.startswith('sky_'):
+            return await self._handle_service_account_token(
+                request, sa_token, call_next)
+
+        # Handle other Bearer tokens (OAuth2 access tokens, etc.)
+        # These requests bypassed OAuth2 proxy, so let the application decide
+        # how to handle them
+        # For now, we'll let them continue through normal processing
+        logger.debug(
+            'Non-SkyPilot Bearer token detected, continuing with normal '
+            'processing')
+        return await call_next(request)
+
+    async def _handle_service_account_token(self, request: fastapi.Request,
+                                            sa_token: str, call_next):
+        """Handle SkyPilot service account tokens."""
+        # Check if service account tokens are enabled
+        sa_enabled = os.environ.get(constants.ENV_VAR_ENABLE_SERVICE_ACCOUNTS,
+                                    'false').lower()
+        if sa_enabled != 'true':
+            return fastapi.responses.JSONResponse(
+                status_code=401,
+                content={'detail': 'Service account authentication disabled'})
+
+        try:
+            # Import here to avoid circular imports
+            # pylint: disable=import-outside-toplevel
+            from sky.users.token_service import token_service
+
+            # Verify and decode JWT token
+            payload = token_service.verify_token(sa_token)
+
+            if payload is None:
+                logger.warning('Service account token verification failed')
+                return fastapi.responses.JSONResponse(
+                    status_code=401,
+                    content={
+                        'detail': 'Invalid or expired service account token'
+                    })
+
+            # Extract user information from JWT payload
+            user_id = payload.get('sub')
+            user_name = payload.get('name')
+            token_id = payload.get('token_id')
+
+            if not user_id or not token_id:
+                logger.warning(
+                    'Invalid token payload: missing user_id or token_id')
+                return fastapi.responses.JSONResponse(
+                    status_code=401,
+                    content={'detail': 'Invalid token payload'})
+
+            # Verify user still exists in database
+            user_info = global_user_state.get_user(user_id)
+            if user_info is None:
+                logger.warning(
+                    f'Service account user {user_id} no longer exists')
+                return fastapi.responses.JSONResponse(
+                    status_code=401,
+                    content={'detail': 'Service account user no longer exists'})
+
+            # Update last used timestamp for token tracking
+            try:
+                global_user_state.update_service_account_token_last_used(
+                    token_id)
+            except Exception as e:  # pylint: disable=broad-except
+                logger.debug(f'Failed to update token last used time: {e}')
+
+            # Set the authenticated user
+            auth_user = models.User(id=user_id,
+                                    name=user_name or user_info.name)
+            request.state.auth_user = auth_user
+
+            # Override user info in request body for service account requests
+            await _override_user_info_in_request_body(request, auth_user)
+
+            logger.debug(f'Authenticated service account: {user_id}')
+
+        except Exception as e:  # pylint: disable=broad-except
+            logger.error(f'Service account authentication failed: {e}',
+                         exc_info=True)
+            return fastapi.responses.JSONResponse(
+                status_code=401,
+                content={
+                    'detail': f'Service account authentication failed: {str(e)}'
+                })
+
+        return await call_next(request)
+
+
 class AuthProxyMiddleware(starlette.middleware.base.BaseHTTPMiddleware):
     """Middleware to handle auth proxy."""
 
@@ -330,7 +441,7 @@ async def lifespan(app: fastapi.FastAPI):  # pylint: disable=redefined-outer-nam
                 request_id=event.id,
                 request_name=event.name,
                 request_body=payloads.RequestBody(),
-                func=event.event_fn,
+                func=event.run_event,
                 schedule_type=requests_lib.ScheduleType.SHORT,
                 is_skypilot_system=True,
             )
@@ -424,6 +535,9 @@ app.add_middleware(
 enable_basic_auth = os.environ.get(constants.ENV_VAR_ENABLE_BASIC_AUTH, 'false')
 if str(enable_basic_auth).lower() == 'true':
     app.add_middleware(BasicAuthMiddleware)
+# Bearer token middleware should always be present to handle service account
+# authentication
+app.add_middleware(BearerTokenMiddleware)
 app.add_middleware(AuthProxyMiddleware)
 app.add_middleware(RequestIDMiddleware)
 app.include_router(jobs_rest.router, prefix='/jobs', tags=['jobs'])
@@ -1339,6 +1453,7 @@ async def health(request: fastapi.Request) -> Dict[str, Any]:
         - commit: str; The commit hash of SkyPilot used for API server.
     """
     user = request.state.auth_user
+    logger.info(f'Health endpoint: request.state.auth_user = {user}')
     return {
         'status': common.ApiServerStatus.HEALTHY.value,
         'api_version': server_constants.API_VERSION,