skypilot-nightly 1.0.0.dev20250905__py3-none-any.whl → 1.0.0.dev20251203__py3-none-any.whl

sky/serve/server/server.py

@@ -10,6 +10,7 @@ from sky.server import common as server_common
 from sky.server import stream_utils
 from sky.server.requests import executor
 from sky.server.requests import payloads
+from sky.server.requests import request_names
 from sky.server.requests import requests as api_requests
 from sky.skylet import constants
 from sky.utils import common
@@ -23,9 +24,9 @@ async def up(
     request: fastapi.Request,
     up_body: payloads.ServeUpBody,
 ) -> None:
-    executor.schedule_request(
+    await executor.schedule_request_async(
         request_id=request.state.request_id,
-        request_name='serve.up',
+        request_name=request_names.RequestName.SERVE_UP,
         request_body=up_body,
         func=core.up,
         schedule_type=api_requests.ScheduleType.LONG,
@@ -38,9 +39,9 @@ async def update(
     request: fastapi.Request,
     update_body: payloads.ServeUpdateBody,
 ) -> None:
-    executor.schedule_request(
+    await executor.schedule_request_async(
         request_id=request.state.request_id,
-        request_name='serve.update',
+        request_name=request_names.RequestName.SERVE_UPDATE,
         request_body=update_body,
         func=core.update,
         schedule_type=api_requests.ScheduleType.SHORT,
@@ -53,9 +54,9 @@ async def down(
     request: fastapi.Request,
     down_body: payloads.ServeDownBody,
 ) -> None:
-    executor.schedule_request(
+    await executor.schedule_request_async(
         request_id=request.state.request_id,
-        request_name='serve.down',
+        request_name=request_names.RequestName.SERVE_DOWN,
         request_body=down_body,
         func=core.down,
         schedule_type=api_requests.ScheduleType.SHORT,
@@ -68,9 +69,9 @@ async def terminate_replica(
     request: fastapi.Request,
     terminate_replica_body: payloads.ServeTerminateReplicaBody,
 ) -> None:
-    executor.schedule_request(
+    await executor.schedule_request_async(
         request_id=request.state.request_id,
-        request_name='serve.terminate_replica',
+        request_name=request_names.RequestName.SERVE_TERMINATE_REPLICA,
         request_body=terminate_replica_body,
         func=core.terminate_replica,
         schedule_type=api_requests.ScheduleType.SHORT,
@@ -83,9 +84,9 @@ async def status(
     request: fastapi.Request,
     status_body: payloads.ServeStatusBody,
 ) -> None:
-    executor.schedule_request(
+    await executor.schedule_request_async(
         request_id=request.state.request_id,
-        request_name='serve.status',
+        request_name=request_names.RequestName.SERVE_STATUS,
         request_body=status_body,
         func=core.status,
         schedule_type=api_requests.ScheduleType.SHORT,
@@ -98,22 +99,23 @@ async def tail_logs(
     request: fastapi.Request, log_body: payloads.ServeLogsBody,
     background_tasks: fastapi.BackgroundTasks
 ) -> fastapi.responses.StreamingResponse:
-    executor.schedule_request(
+    executor.check_request_thread_executor_available()
+    request_task = await executor.prepare_request_async(
         request_id=request.state.request_id,
-        request_name='serve.logs',
+        request_name=request_names.RequestName.SERVE_LOGS,
         request_body=log_body,
         func=core.tail_logs,
         schedule_type=api_requests.ScheduleType.SHORT,
         request_cluster_name=common.SKY_SERVE_CONTROLLER_NAME,
     )
-
-    request_task = await api_requests.get_request_async(request.state.request_id
-                                                       )
-
-    return stream_utils.stream_response(
+    task = executor.execute_request_in_coroutine(request_task)
+    # Cancel the coroutine after the request is done or client disconnects
+    background_tasks.add_task(task.cancel)
+    return stream_utils.stream_response_for_long_request(
         request_id=request_task.request_id,
         logs_path=request_task.log_path,
         background_tasks=background_tasks,
+        kill_request_on_disconnect=False,
     )
 
 
@@ -131,9 +133,9 @@ async def download_logs(
     # We should reuse the original request body, so that the env vars, such as
     # user hash, are kept the same.
     download_logs_body.local_dir = str(logs_dir_on_api_server)
-    executor.schedule_request(
+    await executor.schedule_request_async(
         request_id=request.state.request_id,
-        request_name='serve.sync_down_logs',
+        request_name=request_names.RequestName.SERVE_SYNC_DOWN_LOGS,
         request_body=download_logs_body,
         func=core.sync_down_logs,
         schedule_type=api_requests.ScheduleType.SHORT,

sky/serve/service.py

@@ -13,7 +13,6 @@ from typing import Dict
 
 import filelock
 
-from sky import authentication
 from sky import exceptions
 from sky import global_user_state
 from sky import sky_logging
@@ -21,7 +20,6 @@ from sky import task as task_lib
 from sky.backends import backend_utils
 from sky.backends import cloud_vm_ray_backend
 from sky.data import data_utils
-from sky.jobs import scheduler as jobs_scheduler
 from sky.serve import constants
 from sky.serve import controller
 from sky.serve import load_balancer
@@ -29,9 +27,11 @@ from sky.serve import replica_managers
 from sky.serve import serve_state
 from sky.serve import serve_utils
 from sky.skylet import constants as skylet_constants
+from sky.utils import auth_utils
 from sky.utils import common_utils
 from sky.utils import controller_utils
 from sky.utils import subprocess_utils
+from sky.utils import thread_utils
 from sky.utils import ux_utils
 
 # Use the explicit logger name so that the logger is under the
@@ -66,11 +66,11 @@ def _handle_signal(service_name: str) -> None:
     raise error_type(f'User signal received: {user_signal.value}')
 
 
-def cleanup_storage(task_yaml: str) -> bool:
+def cleanup_storage(yaml_content: str) -> bool:
     """Clean up the storage for the service.
 
     Args:
-        task_yaml: The task yaml file.
+        yaml_content: The yaml content of the service.
 
     Returns:
         True if the storage is cleaned up successfully, False otherwise.
@@ -78,7 +78,7 @@ def cleanup_storage(task_yaml: str) -> bool:
     failed = False
 
     try:
-        task = task_lib.Task.from_yaml(task_yaml)
+        task = task_lib.Task.from_yaml_str(yaml_content)
         backend = cloud_vm_ray_backend.CloudVmRayBackend()
         # Need to re-construct storage object in the controller process
         # because when SkyPilot API server machine sends the yaml config to the
@@ -116,7 +116,7 @@ def cleanup_storage(task_yaml: str) -> bool:
 # NOTE(dev): We don't need to acquire the `with_lock` in replica manager here
 # because we killed all the processes (controller & replica manager) before
 # calling this function.
-def _cleanup(service_name: str) -> bool:
+def _cleanup(service_name: str, pool: bool) -> bool:
     """Clean up all service related resources, i.e. replicas and storage."""
     # Cleanup the HA recovery script first as it is possible that some error
     # was raised when we construct the task object (e.g.,
@@ -124,8 +124,8 @@ def _cleanup(service_name: str) -> bool:
     serve_state.remove_ha_recovery_script(service_name)
     failed = False
     replica_infos = serve_state.get_replica_infos(service_name)
-    info2proc: Dict[replica_managers.ReplicaInfo,
-                    multiprocessing.Process] = dict()
+    info2thr: Dict[replica_managers.ReplicaInfo,
+                   thread_utils.SafeThread] = dict()
     # NOTE(dev): This relies on `sky/serve/serve_utils.py::
     # generate_replica_cluster_name`. Change it if you change the function.
     existing_cluster_names = global_user_state.get_cluster_names_start_with(
@@ -136,9 +136,12 @@ def _cleanup(service_name: str) -> bool:
                         f'{info.replica_id} not found. Might be a failed '
                         'cluster. Skipping.')
             continue
-        p = multiprocessing.Process(target=replica_managers.terminate_cluster,
-                                    args=(info.cluster_name,))
-        info2proc[info] = p
+
+        log_file_name = serve_utils.generate_replica_log_file_name(
+            service_name, info.replica_id)
+        t = thread_utils.SafeThread(target=replica_managers.terminate_cluster,
+                                    args=(info.cluster_name, log_file_name))
+        info2thr[info] = t
         # Set replica status to `SHUTTING_DOWN`
         info.status_property.sky_launch_status = (
             replica_managers.common_utils.ProcessStatus.SUCCEEDED)
@@ -158,32 +161,32 @@ def _cleanup(service_name: str) -> bool:
 
     # Please reference to sky/serve/replica_managers.py::_refresh_process_pool.
     # TODO(tian): Refactor to use the same logic and code.
-    while info2proc:
-        snapshot = list(info2proc.items())
-        for info, p in snapshot:
-            if p.is_alive():
+    while info2thr:
+        snapshot = list(info2thr.items())
+        for info, t in snapshot:
+            if t.is_alive():
                 continue
             if (info.status_property.sky_down_status ==
                     replica_managers.common_utils.ProcessStatus.SCHEDULED):
-                if controller_utils.can_terminate():
+                if controller_utils.can_terminate(pool):
                     try:
-                        p.start()
+                        t.start()
                     except Exception as e:  # pylint: disable=broad-except
                         _set_to_failed_cleanup(info)
-                        logger.error(f'Failed to start process for replica '
+                        logger.error(f'Failed to start thread for replica '
                                      f'{info.replica_id}: {e}')
-                        del info2proc[info]
+                        del info2thr[info]
                     else:
                         info.status_property.sky_down_status = (
                             common_utils.ProcessStatus.RUNNING)
                         serve_state.add_or_update_replica(
                             service_name, info.replica_id, info)
             else:
-                logger.info('Terminate process for replica '
+                logger.info('Terminate thread for replica '
                             f'{info.replica_id} finished.')
-                p.join()
-                del info2proc[info]
-                if p.exitcode == 0:
+                t.join()
+                del info2thr[info]
+                if t.format_exc is None:
                     serve_state.remove_replica(service_name, info.replica_id)
                     logger.info(
                         f'Replica {info.replica_id} terminated successfully.')
@@ -191,19 +194,23 @@ def _cleanup(service_name: str) -> bool:
                     _set_to_failed_cleanup(info)
         time.sleep(3)
 
-    versions = serve_state.get_service_versions(service_name)
-    serve_state.remove_service_versions(service_name)
-
     def cleanup_version_storage(version: int) -> bool:
-        task_yaml: str = serve_utils.generate_task_yaml_file_name(
-            service_name, version)
+        yaml_content = serve_state.get_yaml_content(service_name, version)
+        if yaml_content is None:
+            logger.warning(f'No yaml content found for version {version}')
+            return True
         logger.info(f'Cleaning up storage for version {version}, '
-                    f'task_yaml: {task_yaml}')
-        return cleanup_storage(task_yaml)
+                    f'yaml_content: {yaml_content}')
+        return cleanup_storage(yaml_content)
 
+    versions = serve_state.get_service_versions(service_name)
     if not all(map(cleanup_version_storage, versions)):
         failed = True
 
+    # Cleanup version metadata after all storages are cleaned up, otherwise
+    # the get_yaml_content will return None as all versions are deleted.
+    serve_state.delete_all_versions(service_name)
+
     return failed
 
 
@@ -228,41 +235,39 @@ def _start(service_name: str, tmp_task_yaml: str, job_id: int, entrypoint: str):
     """
     # Generate ssh key pair to avoid race condition when multiple sky.launch
     # are executed at the same time.
-    authentication.get_or_generate_keys()
+    auth_utils.get_or_generate_keys()
 
-    # Initialize database record for the service.
-    task = task_lib.Task.from_yaml(tmp_task_yaml)
-    # Already checked before submit to controller.
-    assert task.service is not None, task
-    service_spec = task.service
-
-    def is_recovery_mode(service_name: str) -> bool:
-        """Check if service exists in database to determine recovery mode.
-        """
-        service = serve_state.get_service_from_name(service_name)
-        return service is not None
-
-    is_recovery = is_recovery_mode(service_name)
+    service = serve_state.get_service_from_name(service_name)
+    is_recovery = service is not None
     logger.info(f'It is a {"first" if not is_recovery else "recovery"} run')
 
+    def _read_yaml_content(yaml_path: str) -> str:
+        with open(os.path.expanduser(yaml_path), 'r', encoding='utf-8') as f:
+            return f.read()
+
     if is_recovery:
-        version = serve_state.get_latest_version(service_name)
-        if version is None:
-            raise ValueError(f'No version found for service {service_name}')
+        yaml_content = service['yaml_content']
+        # Backward compatibility for old service records that
+        # does not dump the yaml content to version database.
+        # TODO(tian): Remove this after 2 minor releases, i.e. 0.13.0.
+        if yaml_content is None:
+            yaml_content = _read_yaml_content(tmp_task_yaml)
     else:
-        version = constants.INITIAL_VERSION
-        # Add initial version information to the service state.
-        serve_state.add_or_update_version(service_name, version, service_spec)
+        yaml_content = _read_yaml_content(tmp_task_yaml)
+
+    # Initialize database record for the service.
+    task = task_lib.Task.from_yaml_str(yaml_content)
+    # Already checked before submit to controller.
+    assert task.service is not None, task
+    service_spec = task.service
 
     service_dir = os.path.expanduser(
         serve_utils.generate_remote_service_dir_name(service_name))
-    service_task_yaml = serve_utils.generate_task_yaml_file_name(
-        service_name, version)
 
     if not is_recovery:
         with filelock.FileLock(controller_utils.get_resources_lock_path()):
-            if not controller_utils.can_start_new_process():
-                cleanup_storage(tmp_task_yaml)
+            if not controller_utils.can_start_new_process(task.service.pool):
+                cleanup_storage(yaml_content)
                 with ux_utils.print_exception_no_traceback():
                     raise RuntimeError(
                         constants.MAX_NUMBER_OF_SERVICES_REACHED_ERROR)
@@ -278,25 +283,24 @@ def _start(service_name: str, tmp_task_yaml: str, job_id: int, entrypoint: str):
                 pool=service_spec.pool,
                 controller_pid=os.getpid(),
                 entrypoint=entrypoint)
-        jobs_scheduler.maybe_schedule_next_jobs()
         # Directly throw an error here. See sky/serve/api.py::up
         # for more details.
         if not success:
-            cleanup_storage(tmp_task_yaml)
+            cleanup_storage(yaml_content)
             with ux_utils.print_exception_no_traceback():
                 raise ValueError(f'Service {service_name} already exists.')
 
         # Create the service working directory.
         os.makedirs(service_dir, exist_ok=True)
 
-        # Copy the tmp task yaml file to the final task yaml file.
-        # This is for the service name conflict case. The _execute will
-        # sync file mounts first and then realized a name conflict. We
-        # don't want the new file mounts to overwrite the old one, so we
-        # sync to a tmp file first and then copy it to the final name
-        # if there is no name conflict.
-        shutil.copy(tmp_task_yaml, service_task_yaml)
+        version = constants.INITIAL_VERSION
+        # Add initial version information to the service state.
+        serve_state.add_or_update_version(service_name, version, service_spec,
+                                          yaml_content)
     else:
+        version = serve_state.get_latest_version(service_name)
+        if version is None:
+            raise ValueError(f'No version found for service {service_name}')
         serve_state.update_service_controller_pid(service_name, os.getpid())
 
     controller_process = None
@@ -328,8 +332,8 @@ def _start(service_name: str, tmp_task_yaml: str, job_id: int, entrypoint: str):
             controller_host = _get_controller_host()
             controller_process = multiprocessing.Process(
                 target=controller.run_controller,
-                args=(service_name, service_spec, service_task_yaml,
-                      controller_host, controller_port))
+                args=(service_name, service_spec, version, controller_host,
+                      controller_port))
             controller_process.start()
 
             if not is_recovery:
@@ -350,8 +354,8 @@ def _start(service_name: str, tmp_task_yaml: str, job_id: int, entrypoint: str):
             # TODO(tian): Probably we could enable multiple ports specified in
             # service spec and we could start multiple load balancers.
             # After that, we will have a mapping from replica port to endpoint.
-            # NOTE(tian): We don't need the load balancer for cluster pool.
-            # Skip the load balancer process for cluster pool.
+            # NOTE(tian): We don't need the load balancer for pool.
+            # Skip the load balancer process for pool.
             if not service_spec.pool:
                 load_balancer_process = multiprocessing.Process(
                     target=ux_utils.RedirectOutputForProcess(
@@ -386,7 +390,19 @@ def _start(service_name: str, tmp_task_yaml: str, job_id: int, entrypoint: str):
         for process in process_to_kill:
             process.join()
 
-        failed = _cleanup(service_name)
+        # Catch any exception here to avoid it kill the service monitoring
+        # process. In which case, the service will not only fail to clean
+        # up, but also cannot be terminated in the future as no process
+        # will handle the user signal anymore. Instead, we catch any error
+        # and set it to FAILED_CLEANUP instead.
+        try:
+            failed = _cleanup(service_name, service_spec.pool)
+        except Exception as e:  # pylint: disable=broad-except
+            logger.error(f'Failed to clean up service {service_name}: {e}')
+            with ux_utils.enable_traceback():
+                logger.error(f'  Traceback: {traceback.format_exc()}')
+            failed = True
+
         if failed:
             serve_state.set_service_status_and_active_versions(
                 service_name, serve_state.ServiceStatus.FAILED_CLEANUP)

sky/serve/service_spec.py

@@ -188,7 +188,7 @@ class SkyServiceSpec:
             with ux_utils.print_exception_no_traceback():
                 raise ValueError('Cannot specify `replica_policy` for cluster '
                                  'pool. Only `workers: <num>` is supported '
-                                 'for cluster pool now.')
+                                 'for pool now.')
 
         simplified_policy_section = config.get('replicas', None)
         workers_config = config.get('workers', None)
@@ -198,7 +198,7 @@ class SkyServiceSpec:
                                  ' Please use one of them.')
         if simplified_policy_section is not None and pool_config:
             with ux_utils.print_exception_no_traceback():
-                raise ValueError('Cannot specify `replicas` for cluster pool. '
+                raise ValueError('Cannot specify `replicas` for pool. '
                                  'Please use `workers` instead.')
         if simplified_policy_section is None:
             simplified_policy_section = workers_config
@@ -266,14 +266,13 @@ class SkyServiceSpec:
         return SkyServiceSpec(**service_config)
 
     @staticmethod
-    def from_yaml(yaml_path: str) -> 'SkyServiceSpec':
-        with open(os.path.expanduser(yaml_path), 'r', encoding='utf-8') as f:
-            config = yaml_utils.safe_load(f)
+    def from_yaml_str(yaml_str: str) -> 'SkyServiceSpec':
+        config = yaml_utils.safe_load(yaml_str)
 
         if isinstance(config, str):
             with ux_utils.print_exception_no_traceback():
                 raise ValueError('YAML loaded as str, not as dict. '
-                                 f'Is it correct? Path: {yaml_path}')
+                                 f'Is it correct? content:\n{yaml_str}')
 
         if config is None:
             config = {}
@@ -281,10 +280,16 @@ class SkyServiceSpec:
         if 'service' not in config:
             with ux_utils.print_exception_no_traceback():
                 raise ValueError('Service YAML must have a "service" section. '
-                                 f'Is it correct? Path: {yaml_path}')
+                                 f'Is it correct? content:\n{yaml_str}')
 
         return SkyServiceSpec.from_yaml_config(config['service'])
 
+    @staticmethod
+    def from_yaml(yaml_path: str) -> 'SkyServiceSpec':
+        with open(os.path.expanduser(yaml_path), 'r', encoding='utf-8') as f:
+            yaml_content = f.read()
+        return SkyServiceSpec.from_yaml_str(yaml_content)
+
     def to_yaml_config(self) -> Dict[str, Any]:
         config: Dict[str, Any] = {}
 
@@ -506,3 +511,36 @@ class SkyServiceSpec:
         if not hasattr(self, '_pool'):
             return False
         return bool(self._pool)
+
+    def copy(self, **override) -> 'SkyServiceSpec':
+        return SkyServiceSpec(
+            readiness_path=override.pop('readiness_path', self._readiness_path),
+            initial_delay_seconds=override.pop('initial_delay_seconds',
+                                               self._initial_delay_seconds),
+            readiness_timeout_seconds=override.pop(
+                'readiness_timeout_seconds', self._readiness_timeout_seconds),
+            min_replicas=override.pop('min_replicas', self._min_replicas),
+            max_replicas=override.pop('max_replicas', self._max_replicas),
+            num_overprovision=override.pop('num_overprovision',
+                                           self._num_overprovision),
+            ports=override.pop('ports', self._ports),
+            target_qps_per_replica=override.pop('target_qps_per_replica',
+                                                self._target_qps_per_replica),
+            post_data=override.pop('post_data', self._post_data),
+            tls_credential=override.pop('tls_credential', self._tls_credential),
+            readiness_headers=override.pop('readiness_headers',
+                                           self._readiness_headers),
+            dynamic_ondemand_fallback=override.pop(
+                'dynamic_ondemand_fallback', self._dynamic_ondemand_fallback),
+            base_ondemand_fallback_replicas=override.pop(
+                'base_ondemand_fallback_replicas',
+                self._base_ondemand_fallback_replicas),
+            spot_placer=override.pop('spot_placer', self._spot_placer),
+            upscale_delay_seconds=override.pop('upscale_delay_seconds',
+                                               self._upscale_delay_seconds),
+            downscale_delay_seconds=override.pop('downscale_delay_seconds',
+                                                 self._downscale_delay_seconds),
+            load_balancing_policy=override.pop('load_balancing_policy',
+                                               self._load_balancing_policy),
+            pool=override.pop('pool', self._pool),
+        )

sky/server/auth/loopback.py

@@ -0,0 +1,38 @@
+"""Shared loopback detection utilities for auth middlewares."""
+
+import ipaddress
+
+import fastapi
+
+from sky import sky_logging
+
+logger = sky_logging.init_logger(__name__)
+
+COMMON_PROXY_HEADERS = [
+    'X-Forwarded-For', 'Forwarded', 'X-Real-IP', 'X-Client-IP',
+    'X-Forwarded-Host', 'X-Forwarded-Proto'
+]
+
+
+def _is_loopback_ip(ip_str: str) -> bool:
+    """Check if an IP address is a loopback address."""
+    try:
+        ip = ipaddress.ip_address(ip_str)
+        return ip.is_loopback
+    except ValueError:
+        return False
+
+
+def is_loopback_request(request: fastapi.Request) -> bool:
+    """Determine if a request is coming from localhost."""
+    if request.client is None:
+        return False
+
+    client_host = request.client.host
+    if client_host == 'localhost' or _is_loopback_ip(client_host):
+        # Additional checks: ensure no forwarding headers are present.
+        # If there are any, assume this traffic went through a proxy.
+        return not any(
+            request.headers.get(header) for header in COMMON_PROXY_HEADERS)
+
+    return False

sky/server/auth/oauth2_proxy.py

@@ -15,7 +15,10 @@ import starlette.middleware.base
 from sky import global_user_state
 from sky import models
 from sky import sky_logging
+from sky.jobs import utils as managed_job_utils
+from sky.server import middleware_utils
 from sky.server.auth import authn
+from sky.server.auth import loopback
 from sky.users import permission
 from sky.utils import common_utils
 
@@ -34,11 +37,12 @@ OAUTH2_PROXY_BASE_URL_ENV_VAR = 'SKYPILOT_AUTH_OAUTH2_PROXY_BASE_URL'
 OAUTH2_PROXY_ENABLED_ENV_VAR = 'SKYPILOT_AUTH_OAUTH2_PROXY_ENABLED'
 
 
+@middleware_utils.websocket_aware
 class OAuth2ProxyMiddleware(starlette.middleware.base.BaseHTTPMiddleware):
     """Middleware to handle authentication by delegating to OAuth2 Proxy."""
 
-    def __init__(self, application: fastapi.FastAPI):
-        super().__init__(application)
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
         self.enabled: bool = (os.getenv(OAUTH2_PROXY_ENABLED_ENV_VAR,
                                         'false') == 'true')
         self.proxy_base: str = ''
@@ -108,6 +112,10 @@ class OAuth2ProxyMiddleware(starlette.middleware.base.BaseHTTPMiddleware):
             # Already authenticated
             return await call_next(request)
 
+        if managed_job_utils.is_consolidation_mode(
+        ) and loopback.is_loopback_request(request):
+            return await call_next(request)
+
         async with aiohttp.ClientSession() as session:
             try:
                 return await self._authenticate(request, call_next, session)
@@ -120,13 +128,10 @@ class OAuth2ProxyMiddleware(starlette.middleware.base.BaseHTTPMiddleware):
 
     async def _authenticate(self, request: fastapi.Request, call_next,
                             session: aiohttp.ClientSession):
-        forwarded_headers = dict(request.headers)
+        forwarded_headers = {}
         auth_url = f'{self.proxy_base}/oauth2/auth'
         forwarded_headers['X-Forwarded-Uri'] = str(request.url).rstrip('/')
-        # Remove content-length and content-type headers and drop request body
-        # to reduce the auth overhead.
-        forwarded_headers.pop('content-length', None)
-        forwarded_headers.pop('content-type', None)
+        forwarded_headers['Host'] = request.url.hostname
         logger.debug(f'authenticate request: {auth_url}, '
                      f'headers: {forwarded_headers}')