skypilot-nightly 1.0.0.dev20250627__py3-none-any.whl → 1.0.0.dev20250628__py3-none-any.whl

sky/server/requests/requests.py

@@ -375,10 +375,29 @@ def managed_job_status_refresh_event():
 
 @dataclasses.dataclass
 class InternalRequestDaemon:
+    """Internal daemon that runs an event in the background."""
+
     id: str
     name: str
     event_fn: Callable[[], None]
 
+    def run_event(self):
+        """Run the event."""
+        while True:
+            with ux_utils.enable_traceback():
+                try:
+                    self.event_fn()
+                    break
+                except Exception:  # pylint: disable=broad-except
+                    # It is OK to fail to run the event, as the event is not
+                    # critical, but we should log the error.
+                    logger.exception(
+                        f'Error running {self.name} event. '
+                        f'Restarting in '
+                        f'{server_constants.DAEMON_RESTART_INTERVAL_SECONDS} '
+                        'seconds...')
+                    time.sleep(server_constants.DAEMON_RESTART_INTERVAL_SECONDS)
+
 
 # Register the events to run in the background.
 INTERNAL_REQUEST_DAEMONS = [

sky/server/rest.py

@@ -129,25 +129,16 @@ def handle_server_unavailable(response: 'requests.Response') -> None:
 
 
 @retry_on_server_unavailable()
-def post(url, data=None, json=None, **kwargs) -> 'requests.Response':
-    """Send a POST request to the API server, retry on server temporarily
+def request(method, url, **kwargs) -> 'requests.Response':
+    """Send a request to the API server, retry on server temporarily
     unavailable."""
-    response = requests.post(url, data=data, json=json, **kwargs)
+    response = requests.request(method, url, **kwargs)
     handle_server_unavailable(response)
     return response
 
 
-@retry_on_server_unavailable()
-def get(url, params=None, **kwargs) -> 'requests.Response':
-    """Send a GET request to the API server, retry on server temporarily
-    unavailable."""
-    response = requests.get(url, params=params, **kwargs)
-    handle_server_unavailable(response)
-    return response
-
-
-def get_without_retry(url, params=None, **kwargs) -> 'requests.Response':
-    """Send a GET request to the API server without retry."""
-    response = requests.get(url, params=params, **kwargs)
+def request_without_retry(method, url, **kwargs) -> 'requests.Response':
+    """Send a request to the API server without retry."""
+    response = requests.request(method, url, **kwargs)
     handle_server_unavailable(response)
     return response

sky/server/server.py

@@ -119,8 +119,11 @@ def _basic_auth_401_response(content: str):
 # TODO(hailong): Remove this function and use request.state.auth_user instead.
 async def _override_user_info_in_request_body(request: fastapi.Request,
                                               auth_user: Optional[models.User]):
+    if auth_user is None:
+        return
+
     body = await request.body()
-    if auth_user and body:
+    if body:
         try:
             original_json = await request.json()
         except json.JSONDecodeError as e:
@@ -228,14 +231,17 @@ class BasicAuthMiddleware(starlette.middleware.base.BaseHTTPMiddleware):
 
     async def dispatch(self, request: fastapi.Request, call_next):
         if request.url.path.startswith('/api/'):
-            # Try to set the auth user from the basic auth header so the
-            # following endpoint handlers can leverage the auth_user info
+            # Try to set the auth user from basic auth
             _try_set_basic_auth_user(request)
             return await call_next(request)
 
         auth_header = request.headers.get('authorization')
-        if not auth_header or not auth_header.lower().startswith('basic '):
-            return _basic_auth_401_response('Invalid basic auth')
+        if not auth_header:
+            return _basic_auth_401_response('Authentication required')
+
+        # Only handle basic auth
+        if not auth_header.lower().startswith('basic '):
+            return _basic_auth_401_response('Invalid authentication method')
 
         # Check username and password
         encoded = auth_header.split(' ', 1)[1]
@@ -267,6 +273,111 @@ class BasicAuthMiddleware(starlette.middleware.base.BaseHTTPMiddleware):
         return await call_next(request)
 
 
+class BearerTokenMiddleware(starlette.middleware.base.BaseHTTPMiddleware):
+    """Middleware to handle Bearer Token Auth (Service Accounts)."""
+
+    async def dispatch(self, request: fastapi.Request, call_next):
+        # Only process requests with Bearer token authorization header
+        auth_header = request.headers.get('authorization')
+        if not auth_header or not auth_header.lower().startswith('bearer '):
+            # No Bearer token, continue with normal processing (OAuth2 cookies,
+            # etc.)
+            return await call_next(request)
+
+        # Extract token
+        sa_token = auth_header.split(' ', 1)[1]
+
+        # Handle SkyPilot service account tokens
+        if sa_token.startswith('sky_'):
+            return await self._handle_service_account_token(
+                request, sa_token, call_next)
+
+        # Handle other Bearer tokens (OAuth2 access tokens, etc.)
+        # These requests bypassed OAuth2 proxy, so let the application decide
+        # how to handle them
+        # For now, we'll let them continue through normal processing
+        logger.debug(
+            'Non-SkyPilot Bearer token detected, continuing with normal '
+            'processing')
+        return await call_next(request)
+
+    async def _handle_service_account_token(self, request: fastapi.Request,
+                                            sa_token: str, call_next):
+        """Handle SkyPilot service account tokens."""
+        # Check if service account tokens are enabled
+        sa_enabled = os.environ.get(constants.ENV_VAR_ENABLE_SERVICE_ACCOUNTS,
+                                    'false').lower()
+        if sa_enabled != 'true':
+            return fastapi.responses.JSONResponse(
+                status_code=401,
+                content={'detail': 'Service account authentication disabled'})
+
+        try:
+            # Import here to avoid circular imports
+            # pylint: disable=import-outside-toplevel
+            from sky.users.token_service import token_service
+
+            # Verify and decode JWT token
+            payload = token_service.verify_token(sa_token)
+
+            if payload is None:
+                logger.warning('Service account token verification failed')
+                return fastapi.responses.JSONResponse(
+                    status_code=401,
+                    content={
+                        'detail': 'Invalid or expired service account token'
+                    })
+
+            # Extract user information from JWT payload
+            user_id = payload.get('sub')
+            user_name = payload.get('name')
+            token_id = payload.get('token_id')
+
+            if not user_id or not token_id:
+                logger.warning(
+                    'Invalid token payload: missing user_id or token_id')
+                return fastapi.responses.JSONResponse(
+                    status_code=401,
+                    content={'detail': 'Invalid token payload'})
+
+            # Verify user still exists in database
+            user_info = global_user_state.get_user(user_id)
+            if user_info is None:
+                logger.warning(
+                    f'Service account user {user_id} no longer exists')
+                return fastapi.responses.JSONResponse(
+                    status_code=401,
+                    content={'detail': 'Service account user no longer exists'})
+
+            # Update last used timestamp for token tracking
+            try:
+                global_user_state.update_service_account_token_last_used(
+                    token_id)
+            except Exception as e:  # pylint: disable=broad-except
+                logger.debug(f'Failed to update token last used time: {e}')
+
+            # Set the authenticated user
+            auth_user = models.User(id=user_id,
+                                    name=user_name or user_info.name)
+            request.state.auth_user = auth_user
+
+            # Override user info in request body for service account requests
+            await _override_user_info_in_request_body(request, auth_user)
+
+            logger.debug(f'Authenticated service account: {user_id}')
+
+        except Exception as e:  # pylint: disable=broad-except
+            logger.error(f'Service account authentication failed: {e}',
+                         exc_info=True)
+            return fastapi.responses.JSONResponse(
+                status_code=401,
+                content={
+                    'detail': f'Service account authentication failed: {str(e)}'
+                })
+
+        return await call_next(request)
+
+
 class AuthProxyMiddleware(starlette.middleware.base.BaseHTTPMiddleware):
     """Middleware to handle auth proxy."""
 
@@ -330,7 +441,7 @@ async def lifespan(app: fastapi.FastAPI):  # pylint: disable=redefined-outer-nam
                 request_id=event.id,
                 request_name=event.name,
                 request_body=payloads.RequestBody(),
-                func=event.event_fn,
+                func=event.run_event,
                 schedule_type=requests_lib.ScheduleType.SHORT,
                 is_skypilot_system=True,
             )
@@ -424,6 +535,9 @@ app.add_middleware(
 enable_basic_auth = os.environ.get(constants.ENV_VAR_ENABLE_BASIC_AUTH, 'false')
 if str(enable_basic_auth).lower() == 'true':
     app.add_middleware(BasicAuthMiddleware)
+# Bearer token middleware should always be present to handle service account
+# authentication
+app.add_middleware(BearerTokenMiddleware)
 app.add_middleware(AuthProxyMiddleware)
 app.add_middleware(RequestIDMiddleware)
 app.include_router(jobs_rest.router, prefix='/jobs', tags=['jobs'])
@@ -1339,6 +1453,7 @@ async def health(request: fastapi.Request) -> Dict[str, Any]:
         - commit: str; The commit hash of SkyPilot used for API server.
     """
     user = request.state.auth_user
+    logger.info(f'Health endpoint: request.state.auth_user = {user}')
     return {
         'status': common.ApiServerStatus.HEALTHY.value,
         'api_version': server_constants.API_VERSION,

sky/skylet/constants.py

@@ -346,6 +346,11 @@ API_SERVER_CREATION_LOCK_PATH = '~/.sky/api_server/.creation.lock'
 # API server.
 SKY_API_SERVER_URL_ENV_VAR = f'{SKYPILOT_ENV_VAR_PREFIX}API_SERVER_ENDPOINT'
 
+# The name for the environment variable that stores the SkyPilot service
+# account token on client side.
+SERVICE_ACCOUNT_TOKEN_ENV_VAR = (
+    f'{SKYPILOT_ENV_VAR_PREFIX}SERVICE_ACCOUNT_TOKEN')
+
 # SkyPilot environment variables
 SKYPILOT_NUM_NODES = f'{SKYPILOT_ENV_VAR_PREFIX}NUM_NODES'
 SKYPILOT_NODE_IPS = f'{SKYPILOT_ENV_VAR_PREFIX}NODE_IPS'
@@ -424,6 +429,7 @@ ENV_VAR_DB_CONNECTION_URI = (f'{SKYPILOT_ENV_VAR_PREFIX}DB_CONNECTION_URI')
 # authentication is enabled in the API server.
 ENV_VAR_ENABLE_BASIC_AUTH = 'ENABLE_BASIC_AUTH'
 SKYPILOT_INITIAL_BASIC_AUTH = 'SKYPILOT_INITIAL_BASIC_AUTH'
+ENV_VAR_ENABLE_SERVICE_ACCOUNTS = 'ENABLE_SERVICE_ACCOUNTS'
 
 SKYPILOT_DEFAULT_WORKSPACE = 'default'
 

sky/skypilot_config.py

@@ -369,6 +369,34 @@ def get_nested(keys: Tuple[str, ...],
         disallowed_override_keys=None)
 
 
+def get_effective_region_config(
+        cloud: str,
+        keys: Tuple[str, ...],
+        region: Optional[str] = None,
+        default_value: Optional[Any] = None,
+        override_configs: Optional[Dict[str, Any]] = None) -> Any:
+    """Returns the nested key value by reading from config
+    Order to get the property_name value:
+    1. if region is specified,
+       try to get the value from <cloud>/<region_key>/<region>/keys
+    2. if no region or no override,
+       try to get it at the cloud level <cloud>/keys
+    3. if not found at cloud level,
+       return either default_value if specified or None
+
+    Note: This function currently only supports getting region-specific
+    config from "kubernetes" cloud. For other clouds, this function behaves
+    identically to get_nested().
+    """
+    return config_utils.get_cloud_config_value_from_dict(
+        dict_config=_get_loaded_config(),
+        cloud=cloud,
+        keys=keys,
+        region=region,
+        default_value=default_value,
+        override_configs=override_configs)
+
+
 def get_workspace_cloud(cloud: str,
                         workspace: Optional[str] = None) -> config_utils.Config:
     """Returns the workspace config."""
@@ -477,10 +505,10 @@ def overlay_skypilot_config(
 def safe_reload_config() -> None:
     """Reloads the config, safe to be called concurrently."""
     with filelock.FileLock(get_skypilot_config_lock_path()):
-        _reload_config()
+        reload_config()
 
 
-def _reload_config() -> None:
+def reload_config() -> None:
     internal_config_path = os.environ.get(ENV_VAR_SKYPILOT_CONFIG)
     if internal_config_path is not None:
         # {ENV_VAR_SKYPILOT_CONFIG} is used internally.
@@ -641,7 +669,7 @@ def loaded_config_path_serialized() -> Optional[str]:
 
 
 # Load on import, synchronization is guaranteed by python interpreter.
-_reload_config()
+reload_config()
 
 
 def loaded() -> bool:
@@ -864,4 +892,4 @@ def update_api_server_config_no_lock(config: config_utils.Config) -> None:
             config_map_utils.patch_configmap_with_config(
                 config, global_config_path)
 
-    _reload_config()
+    reload_config()

sky/users/permission.py

@@ -265,6 +265,35 @@ class PermissionService:
                      f'workspace={workspace_name}, result={result}')
         return result
 
+    def check_service_account_token_permission(self, user_id: str,
+                                               token_owner_id: str,
+                                               action: str) -> bool:
+        """Check service account token permission.
+
+        This method checks if a user has permission to perform an action on
+        a service account token owned by another user.
+
+        Args:
+            user_id: The ID of the user requesting the action
+            token_owner_id: The ID of the user who owns the token
+            action: The action being performed (e.g., 'delete', 'view')
+
+        Returns:
+            True if the user has permission, False otherwise
+        """
+        del action
+        # Users can always manage their own tokens
+        if user_id == token_owner_id:
+            return True
+
+        # Check if user has admin role (admins can manage any token)
+        user_roles = self.get_user_roles(user_id)
+        if rbac.RoleName.ADMIN.value in user_roles:
+            return True
+
+        # Regular users cannot manage tokens owned by others
+        return False
+
     def add_workspace_policy(self, workspace_name: str,
                              users: List[str]) -> None:
         """Add workspace policy.