skypilot-nightly 1.0.0.dev20250413__py3-none-any.whl → 1.0.0.dev20250421__py3-none-any.whl

sky/serve/serve_utils.py

@@ -13,7 +13,7 @@ import threading
 import time
 import typing
 from typing import (Any, Callable, DefaultDict, Dict, Generic, Iterator, List,
-                    Optional, TextIO, Type, TypeVar)
+                    Optional, TextIO, Type, TypeVar, Union)
 import uuid
 
 import colorama
@@ -81,6 +81,38 @@ class ServiceComponent(enum.Enum):
     REPLICA = 'replica'
 
 
+@dataclasses.dataclass
+class ServiceComponentTarget:
+    """Represents a target service component with an optional replica ID.
+    """
+    component: ServiceComponent
+    replica_id: Optional[int] = None
+
+    def __init__(self,
+                 component: Union[str, ServiceComponent],
+                 replica_id: Optional[int] = None):
+        if isinstance(component, str):
+            component = ServiceComponent(component)
+        self.component = component
+        self.replica_id = replica_id
+
+    def __post_init__(self):
+        """Validate that replica_id is only provided for REPLICA component."""
+        if (self.component
+                == ServiceComponent.REPLICA) != (self.replica_id is None):
+            raise ValueError(
+                'replica_id must be specified if and only if component is '
+                'REPLICA.')
+
+    def __hash__(self) -> int:
+        return hash((self.component, self.replica_id))
+
+    def __str__(self) -> str:
+        if self.component == ServiceComponent.REPLICA:
+            return f'{self.component.value}-{self.replica_id}'
+        return self.component.value
+
+
 class UserSignal(enum.Enum):
     """User signal to send to controller.
 

sky/serve/server/core.py

@@ -1,9 +1,11 @@
 """SkyServe core APIs."""
+import pathlib
 import re
 import signal
 import tempfile
 import threading
-from typing import Any, Dict, List, Optional, Tuple, Union
+import typing
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
 
 import colorama
 
@@ -29,6 +31,9 @@ from sky.utils import rich_utils
 from sky.utils import subprocess_utils
 from sky.utils import ux_utils
 
+if typing.TYPE_CHECKING:
+    from sky.backends import cloud_vm_ray_backend
+
 logger = sky_logging.init_logger(__name__)
 
 
@@ -64,6 +69,41 @@ def _rewrite_tls_credential_paths_and_get_tls_env_vars(
     return tls_template_vars
 
 
+def _get_all_replica_targets(
+    service_name: str, backend: backends.CloudVmRayBackend,
+    handle: backends.CloudVmRayResourceHandle
+) -> Set[serve_utils.ServiceComponentTarget]:
+    """Helper function to get targets for all live replicas."""
+    code = serve_utils.ServeCodeGen.get_service_status([service_name])
+    returncode, serve_status_payload, stderr = backend.run_on_head(
+        handle,
+        code,
+        require_outputs=True,
+        stream_logs=False,
+        separate_stderr=True)
+
+    try:
+        subprocess_utils.handle_returncode(returncode,
+                                           code,
+                                           'Failed to fetch services',
+                                           stderr,
+                                           stream_logs=True)
+    except exceptions.CommandError as e:
+        raise RuntimeError(e.error_msg) from e
+
+    service_records = serve_utils.load_service_status(serve_status_payload)
+    if not service_records:
+        raise ValueError(f'Service {service_name!r} not found.')
+    assert len(service_records) == 1
+    service_record = service_records[0]
+
+    return {
+        serve_utils.ServiceComponentTarget(serve_utils.ServiceComponent.REPLICA,
+                                           replica_info['replica_id'])
+        for replica_info in service_record['replica_info']
+    }
+
+
 @usage_lib.entrypoint
 def up(
     task: 'sky.Task',
@@ -179,14 +219,17 @@ def up(
         # whether the service is already running. If the id is the same
         # with the current job id, we know the service is up and running
         # for the first time; otherwise it is a name conflict.
-        idle_minutes_to_autostop = constants.CONTROLLER_IDLE_MINUTES_TO_AUTOSTOP
+        controller_idle_minutes_to_autostop, controller_down = (
+            controller_utils.get_controller_autostop_config(
+                controller=controller_utils.Controllers.SKY_SERVE_CONTROLLER))
         # Since the controller may be shared among multiple users, launch the
         # controller with the API server's user hash.
         with common.with_server_user_hash():
             controller_job_id, controller_handle = execution.launch(
                 task=controller_task,
                 cluster_name=controller_name,
-                idle_minutes_to_autostop=idle_minutes_to_autostop,
+                idle_minutes_to_autostop=controller_idle_minutes_to_autostop,
+                down=controller_down,
                 retry_until_up=True,
                 _disable_controller_check=True,
             )
@@ -682,11 +725,14 @@ def status(
     return service_records
 
 
+ServiceComponentOrStr = Union[str, serve_utils.ServiceComponent]
+
+
 @usage_lib.entrypoint
 def tail_logs(
     service_name: str,
     *,
-    target: Union[str, serve_utils.ServiceComponent],
+    target: ServiceComponentOrStr,
     replica_id: Optional[int] = None,
     follow: bool = True,
 ) -> None:
@@ -740,10 +786,11 @@ def tail_logs(
             with ux_utils.print_exception_no_traceback():
                 raise ValueError('`replica_id` must be None when using '
                                  'target=CONTROLLER/LOAD_BALANCER.')
+
+    controller_type = controller_utils.Controllers.SKY_SERVE_CONTROLLER
     handle = backend_utils.is_controller_accessible(
-        controller=controller_utils.Controllers.SKY_SERVE_CONTROLLER,
-        stopped_message=(controller_utils.Controllers.SKY_SERVE_CONTROLLER.
-                         value.default_hint_if_non_existent))
+        controller=controller_type,
+        stopped_message=controller_type.value.default_hint_if_non_existent)
 
     backend = backend_utils.get_backend_from_handle(handle)
     assert isinstance(backend, backends.CloudVmRayBackend), backend
@@ -772,3 +819,141 @@ def tail_logs(
                         stream_logs=True,
                         process_stream=False,
                         ssh_mode=command_runner.SshMode.INTERACTIVE)
+
+
+@usage_lib.entrypoint
+def sync_down_logs(
+    service_name: str,
+    *,
+    local_dir: str,
+    targets: Union[ServiceComponentOrStr, List[ServiceComponentOrStr],
+                   None] = None,
+    replica_ids: Optional[List[int]] = None,
+) -> str:
+    """Sync down logs from the controller for the given service.
+
+    This function is called by the server endpoint. It gathers logs from the
+    controller, load balancer, and/or replicas and places them in a directory
+    under the user's log space on the API server filesystem.
+
+    Args:
+        service_name: The name of the service to download logs from.
+        local_dir: The local directory to save the logs to.
+        targets: Which component(s) to download logs for. If None or empty,
+            means download all logs (controller, load-balancer, all replicas).
+            Can be a string (e.g. "controller"), or a `ServiceComponent` object,
+            or a list of them for multiple components. Currently accepted
+            values:
+                - "controller"/ServiceComponent.CONTROLLER
+                - "load_balancer"/ServiceComponent.LOAD_BALANCER
+                - "replica"/ServiceComponent.REPLICA
+        replica_ids: The list of replica IDs to download logs from, specified
+            when target includes `ServiceComponent.REPLICA`. If target includes
+            `ServiceComponent.REPLICA` but this is None/empty, logs for all
+            replicas will be downloaded.
+
+    Returns:
+        A dict mapping component names to local paths where the logs were synced
+        down to.
+
+    Raises:
+        RuntimeError: If fails to gather logs or fails to rsync from the
+          controller.
+        sky.exceptions.ClusterNotUpError: If the controller is not up.
+        ValueError: Arguments not valid.
+    """
+    # Step 0) get the controller handle
+    with rich_utils.safe_status(
+            ux_utils.spinner_message('Checking service status...')):
+        controller_type = controller_utils.Controllers.SKY_SERVE_CONTROLLER
+        handle = backend_utils.is_controller_accessible(
+            controller=controller_type,
+            stopped_message=controller_type.value.default_hint_if_non_existent)
+        backend: backends.CloudVmRayBackend = (
+            backend_utils.get_backend_from_handle(handle))
+
+    requested_components: Set[serve_utils.ServiceComponent] = set()
+    if not targets:
+        # No targets specified -> request all components
+        requested_components = {
+            serve_utils.ServiceComponent.CONTROLLER,
+            serve_utils.ServiceComponent.LOAD_BALANCER,
+            serve_utils.ServiceComponent.REPLICA
+        }
+    else:
+        # Parse provided targets
+        if isinstance(targets, (str, serve_utils.ServiceComponent)):
+            requested_components = {serve_utils.ServiceComponent(targets)}
+        else:  # list
+            requested_components = {
+                serve_utils.ServiceComponent(t) for t in targets
+            }
+
+    normalized_targets: Set[serve_utils.ServiceComponentTarget] = set()
+    if serve_utils.ServiceComponent.CONTROLLER in requested_components:
+        normalized_targets.add(
+            serve_utils.ServiceComponentTarget(
+                serve_utils.ServiceComponent.CONTROLLER))
+    if serve_utils.ServiceComponent.LOAD_BALANCER in requested_components:
+        normalized_targets.add(
+            serve_utils.ServiceComponentTarget(
+                serve_utils.ServiceComponent.LOAD_BALANCER))
+    if serve_utils.ServiceComponent.REPLICA in requested_components:
+        with rich_utils.safe_status(
+                ux_utils.spinner_message('Getting live replica infos...')):
+            replica_targets = _get_all_replica_targets(service_name, backend,
+                                                       handle)
+        if not replica_ids:
+            # Replica target requested but no specific IDs
+            # -> Get all replica logs
+            normalized_targets.update(replica_targets)
+        else:
+            # Replica target requested with specific IDs
+            requested_replica_targets = [
+                serve_utils.ServiceComponentTarget(
+                    serve_utils.ServiceComponent.REPLICA, rid)
+                for rid in replica_ids
+            ]
+            for target in requested_replica_targets:
+                if target not in replica_targets:
+                    logger.warning(f'Replica ID {target.replica_id} not found '
+                                   f'for {service_name}. Skipping...')
+                else:
+                    normalized_targets.add(target)
+
+    def sync_down_logs_by_target(target: serve_utils.ServiceComponentTarget):
+        component = target.component
+        # We need to set one side of the pipe to a logs stream, and the other
+        # side to a file.
+        log_path = str(pathlib.Path(local_dir) / f'{target}.log')
+        stream_logs_code: str
+
+        if component == serve_utils.ServiceComponent.CONTROLLER:
+            stream_logs_code = (
+                serve_utils.ServeCodeGen.stream_serve_process_logs(
+                    service_name, stream_controller=True, follow=False))
+        elif component == serve_utils.ServiceComponent.LOAD_BALANCER:
+            stream_logs_code = (
+                serve_utils.ServeCodeGen.stream_serve_process_logs(
+                    service_name, stream_controller=False, follow=False))
+        elif component == serve_utils.ServiceComponent.REPLICA:
+            replica_id = target.replica_id
+            assert replica_id is not None, service_name
+            stream_logs_code = serve_utils.ServeCodeGen.stream_replica_logs(
+                service_name, replica_id, follow=False)
+        else:
+            assert False, component
+
+        # Refer to the notes in
+        # sky/backends/cloud_vm_ray_backend.py::CloudVmRayBackend::tail_logs.
+        backend.run_on_head(handle,
+                            stream_logs_code,
+                            stream_logs=False,
+                            process_stream=False,
+                            ssh_mode=command_runner.SshMode.INTERACTIVE,
+                            log_path=log_path)
+
+    subprocess_utils.run_in_parallel(sync_down_logs_by_target,
+                                     list(normalized_targets))
+
+    return local_dir

sky/serve/server/server.py

@@ -1,13 +1,17 @@
 """Rest APIs for SkyServe."""
 
+import pathlib
+
 import fastapi
 
 from sky import sky_logging
 from sky.serve.server import core
+from sky.server import common as server_common
 from sky.server import stream_utils
 from sky.server.requests import executor
 from sky.server.requests import payloads
 from sky.server.requests import requests as api_requests
+from sky.skylet import constants
 from sky.utils import common
 
 logger = sky_logging.init_logger(__name__)
@@ -110,3 +114,27 @@ async def tail_logs(
         logs_path=request_task.log_path,
         background_tasks=background_tasks,
     )
+
+
+@router.post('/sync-down-logs')
+async def download_logs(
+    request: fastapi.Request,
+    download_logs_body: payloads.ServeDownloadLogsBody,
+) -> None:
+    user_hash = download_logs_body.env_vars[constants.USER_ID_ENV_VAR]
+    timestamp = sky_logging.get_run_timestamp()
+    logs_dir_on_api_server = (
+        pathlib.Path(server_common.api_server_user_logs_dir_prefix(user_hash)) /
+        'service' / f'{download_logs_body.service_name}_{timestamp}')
+    logs_dir_on_api_server.mkdir(parents=True, exist_ok=True)
+    # We should reuse the original request body, so that the env vars, such as
+    # user hash, are kept the same.
+    download_logs_body.local_dir = str(logs_dir_on_api_server)
+    executor.schedule_request(
+        request_id=request.state.request_id,
+        request_name='serve.sync_down_logs',
+        request_body=download_logs_body,
+        func=core.sync_down_logs,
+        schedule_type=api_requests.ScheduleType.SHORT,
+        request_cluster_name=common.SKY_SERVE_CONTROLLER_NAME,
+    )

sky/server/common.py

@@ -17,6 +17,7 @@ import uuid
 import colorama
 import filelock
 
+import sky
 from sky import exceptions
 from sky import sky_logging
 from sky import skypilot_config
@@ -57,12 +58,36 @@ RETRY_COUNT_ON_TIMEOUT = 3
 # (e.g. in high contention env) and we will exit eagerly if server exit.
 WAIT_APISERVER_START_TIMEOUT_SEC = 60
 
-SKY_API_VERSION_WARNING = (
-    f'{colorama.Fore.YELLOW}SkyPilot API server is too old: '
-    f'v{{server_version}} (client version is v{{client_version}}). '
-    'Please restart the SkyPilot API server with: '
+_VERSION_INFO = (
+    f'{colorama.Style.RESET_ALL}'
+    f'{colorama.Style.DIM}'
+    'client version: v{client_version} (API version: v{client_api_version})\n'
+    'server version: v{server_version} (API version: v{server_api_version})'
+    f'{colorama.Style.RESET_ALL}')
+_LOCAL_SERVER_VERSION_MISMATCH_WARNING = (
+    f'{colorama.Fore.YELLOW}Client and local API server version mismatch:\n'
+    '{version_info}\n'
+    f'{colorama.Fore.YELLOW}Please restart the SkyPilot API server with:\n'
     'sky api stop; sky api start'
     f'{colorama.Style.RESET_ALL}')
+_CLIENT_TOO_OLD_WARNING = (
+    f'{colorama.Fore.YELLOW}Your SkyPilot client is too old:\n'
+    '{version_info}\n'
+    f'{colorama.Fore.YELLOW}Upgrade your client with:\n'
+    '{command}'
+    f'{colorama.Style.RESET_ALL}')
+_REMOTE_SERVER_TOO_OLD_WARNING = (
+    f'{colorama.Fore.YELLOW}SkyPilot API server is too old:\n'
+    '{version_info}\n'
+    f'{colorama.Fore.YELLOW}Contact your administrator to upgrade the '
+    'remote API server or downgrade your local client with:\n'
+    '{command}\n'
+    f'{colorama.Style.RESET_ALL}')
+# Parse local API version eargly to catch version format errors.
+_LOCAL_API_VERSION: int = int(server_constants.API_VERSION)
+# SkyPilot dev version.
+_DEV_VERSION = '1.0.0-dev0'
+
 RequestId = str
 ApiVersion = Optional[str]
 
@@ -78,7 +103,9 @@ class ApiServerStatus(enum.Enum):
 @dataclasses.dataclass
 class ApiServerInfo:
     status: ApiServerStatus
-    api_version: ApiVersion
+    api_version: ApiVersion = None
+    version: Optional[str] = None
+    commit: Optional[str] = None
 
 
 def get_api_cookie_jar() -> requests.cookies.RequestsCookieJar:
@@ -137,37 +164,35 @@ def get_api_server_status(endpoint: Optional[str] = None) -> ApiServerInfo:
                 try:
                     result = response.json()
                     api_version = result.get('api_version')
-                    if api_version is None:
+                    version = result.get('version')
+                    commit = result.get('commit')
+                    server_info = ApiServerInfo(status=ApiServerStatus.HEALTHY,
+                                                api_version=api_version,
+                                                version=version,
+                                                commit=commit)
+                    if api_version is None or version is None or commit is None:
                         logger.warning(f'API server response missing '
                                        f'version info. {server_url} may '
                                        f'not be running SkyPilot API server.')
-                        return ApiServerInfo(status=ApiServerStatus.UNHEALTHY,
-                                             api_version=None)
-                    if api_version == server_constants.API_VERSION:
-                        return ApiServerInfo(status=ApiServerStatus.HEALTHY,
-                                             api_version=api_version)
-                    return ApiServerInfo(
-                        status=ApiServerStatus.VERSION_MISMATCH,
-                        api_version=api_version)
+                        server_info.status = ApiServerStatus.UNHEALTHY
+                    elif api_version != server_constants.API_VERSION:
+                        server_info.status = ApiServerStatus.VERSION_MISMATCH
+                    return server_info
                 except (json.JSONDecodeError, AttributeError) as e:
                     logger.warning('Failed to parse API server response: '
                                    f'{str(e)}')
-                    return ApiServerInfo(status=ApiServerStatus.UNHEALTHY,
-                                         api_version=None)
+                    return ApiServerInfo(status=ApiServerStatus.UNHEALTHY)
             else:
-                return ApiServerInfo(status=ApiServerStatus.UNHEALTHY,
-                                     api_version=None)
+                return ApiServerInfo(status=ApiServerStatus.UNHEALTHY)
         except requests.exceptions.Timeout:
             if time_out_try_count == RETRY_COUNT_ON_TIMEOUT:
-                return ApiServerInfo(status=ApiServerStatus.UNHEALTHY,
-                                     api_version=None)
+                return ApiServerInfo(status=ApiServerStatus.UNHEALTHY)
             time_out_try_count += 1
             continue
         except requests.exceptions.ConnectionError:
-            return ApiServerInfo(status=ApiServerStatus.UNHEALTHY,
-                                 api_version=None)
+            return ApiServerInfo(status=ApiServerStatus.UNHEALTHY)
 
-    return ApiServerInfo(status=ApiServerStatus.UNHEALTHY, api_version=None)
+    return ApiServerInfo(status=ApiServerStatus.UNHEALTHY)
 
 
 def handle_request_error(response: 'requests.Response') -> None:
@@ -227,6 +252,7 @@ def _start_api_server(deploy: bool = False,
 
         if foreground:
             # Replaces the current process with the API server
+            os.environ[constants.ENV_VAR_IS_SKYPILOT_SERVER] = 'true'
             os.execvp(args[0], args)
 
         log_path = os.path.expanduser(constants.API_SERVER_LOGS)
@@ -237,7 +263,12 @@ def _start_api_server(deploy: bool = False,
         # If this is called from a CLI invocation, we need
         # start_new_session=True so that SIGINT on the CLI will not also kill
         # the API server.
-        proc = subprocess.Popen(cmd, shell=True, start_new_session=True)
+        server_env = os.environ.copy()
+        server_env[constants.ENV_VAR_IS_SKYPILOT_SERVER] = 'true'
+        proc = subprocess.Popen(cmd,
+                                shell=True,
+                                start_new_session=True,
+                                env=server_env)
 
         start_time = time.time()
         while True:
@@ -247,21 +278,40 @@ def _start_api_server(deploy: bool = False,
                     raise RuntimeError(
                         'SkyPilot API server process exited unexpectedly.\n'
                         f'View logs at: {constants.API_SERVER_LOGS}')
-            api_server_info = get_api_server_status()
-            assert api_server_info.status != ApiServerStatus.VERSION_MISMATCH, (
-                f'API server version mismatch when starting the server. '
-                f'Server version: {api_server_info.api_version} '
-                f'Client version: {server_constants.API_VERSION}')
-            if api_server_info.status == ApiServerStatus.HEALTHY:
+            try:
+                check_server_healthy()
+            except exceptions.APIVersionMismatchError:
+                raise
+            except Exception as e:  # pylint: disable=broad-except
+                if time.time() - start_time >= WAIT_APISERVER_START_TIMEOUT_SEC:
+                    with ux_utils.print_exception_no_traceback():
+                        raise RuntimeError(
+                            'Failed to start SkyPilot API server at '
+                            f'{get_server_url(host)}'
+                            '\nView logs at: '
+                            f'{constants.API_SERVER_LOGS}') from e
+                time.sleep(0.5)
+            else:
                 break
-            elif time.time() - start_time >= WAIT_APISERVER_START_TIMEOUT_SEC:
-                with ux_utils.print_exception_no_traceback():
-                    raise RuntimeError(
-                        'Failed to start SkyPilot API server at '
-                        f'{get_server_url(host)}'
-                        f'\nView logs at: {constants.API_SERVER_LOGS}')
-            time.sleep(0.5)
-        logger.info(ux_utils.finishing_message('SkyPilot API server started.'))
+
+        dashboard_msg = (f'Dashboard: {get_server_url(host)}/dashboard')
+        api_server_info = get_api_server_status(get_server_url(host))
+        if api_server_info.version == _DEV_VERSION:
+            dashboard_msg += (
+                f'\n{colorama.Style.RESET_ALL}{ux_utils.INDENT_SYMBOL}'
+                f'{colorama.Fore.YELLOW}')
+            if not os.path.isdir(server_constants.DASHBOARD_DIR):
+                dashboard_msg += (
+                    'Dashboard is not built, '
+                    'to build: npm --prefix sky/dashboard run build')
+            else:
+                dashboard_msg += (
+                    'Dashboard may be stale when installed from source, '
+                    'to rebuild: npm --prefix sky/dashboard run build')
+            dashboard_msg += f'{colorama.Style.RESET_ALL}'
+        logger.info(
+            ux_utils.finishing_message(
+                f'SkyPilot API server started. {dashboard_msg}'))
 
 
 def check_server_healthy(endpoint: Optional[str] = None,) -> None:
@@ -279,16 +329,70 @@ def check_server_healthy(endpoint: Optional[str] = None,) -> None:
     api_server_info = get_api_server_status(endpoint)
     api_server_status = api_server_info.status
     if api_server_status == ApiServerStatus.VERSION_MISMATCH:
+        sv = api_server_info.api_version
+        assert sv is not None, 'Server API version is None'
+        try:
+            server_is_older = int(sv) < _LOCAL_API_VERSION
+        except ValueError:
+            # Raised when the server version using an unknown scheme.
+            # Version compatibility checking is expected to handle all legacy
+            # cases so we safely assume the server is newer when the version
+            # scheme is unknown.
+            logger.debug('API server version using unknown scheme: %s', sv)
+            server_is_older = False
+        version_info = _get_version_info_hint(api_server_info)
+        if is_api_server_local():
+            # For local server, just hint user to restart the server to get
+            # a consistent version.
+            msg = _LOCAL_SERVER_VERSION_MISMATCH_WARNING.format(
+                version_info=version_info)
+        else:
+            assert api_server_info.version is not None, 'Server version is None'
+            if server_is_older:
+                msg = _REMOTE_SERVER_TOO_OLD_WARNING.format(
+                    version_info=version_info,
+                    command=_install_server_version_command(api_server_info))
+            else:
+                msg = _CLIENT_TOO_OLD_WARNING.format(
+                    version_info=version_info,
+                    command=_install_server_version_command(api_server_info))
         with ux_utils.print_exception_no_traceback():
-            raise RuntimeError(
-                SKY_API_VERSION_WARNING.format(
-                    server_version=api_server_info.api_version,
-                    client_version=server_constants.API_VERSION))
+            raise exceptions.APIVersionMismatchError(msg)
     elif api_server_status == ApiServerStatus.UNHEALTHY:
         with ux_utils.print_exception_no_traceback():
             raise exceptions.ApiServerConnectionError(endpoint)
 
 
+def _get_version_info_hint(server_info: ApiServerInfo) -> str:
+    assert server_info.version is not None, 'Server version is None'
+    assert server_info.commit is not None, 'Server commit is None'
+    sv = server_info.version
+    cv = sky.__version__
+    if server_info.version == _DEV_VERSION:
+        sv = f'{sv} with commit {server_info.commit}'
+    if cv == _DEV_VERSION:
+        cv = f'{cv} with commit {sky.__commit__}'
+    return _VERSION_INFO.format(client_version=cv,
+                                server_version=sv,
+                                client_api_version=server_constants.API_VERSION,
+                                server_api_version=server_info.api_version)
+
+
+def _install_server_version_command(server_info: ApiServerInfo) -> str:
+    assert server_info.version is not None, 'Server version is None'
+    assert server_info.commit is not None, 'Server commit is None'
+    if server_info.version == _DEV_VERSION:
+        # Dev build without valid version.
+        return ('pip install git+https://github.com/skypilot-org/skypilot@'
+                f'{server_info.commit}')
+    elif 'dev' in server_info.version:
+        # Nightly version.
+        return f'pip install -U "skypilot-nightly=={server_info.version}"'
+    else:
+        # Stable version.
+        return f'pip install -U "skypilot=={server_info.version}"'
+
+
 def check_server_healthy_or_start_fn(deploy: bool = False,
                                      host: str = '127.0.0.1',
                                      foreground: bool = False):
@@ -436,6 +540,12 @@ def reload_for_new_request(client_entrypoint: Optional[str],
                            client_command: Optional[str],
                            using_remote_api_server: bool):
     """Reload modules, global variables, and usage message for a new request."""
+    # This should be called first to make sure the logger is up-to-date.
+    sky_logging.reload_logger()
+
+    # Reload the skypilot config to make sure the latest config is used.
+    skypilot_config.safe_reload_config()
+
     # Reset the client entrypoint and command for the usage message.
     common_utils.set_client_status(
         client_entrypoint=client_entrypoint,
@@ -452,11 +562,6 @@ def reload_for_new_request(client_entrypoint: Optional[str],
     # latest information in the context, e.g. client entrypoint and run id.
     usage_lib.messages.reset(usage_lib.MessageType.USAGE)
 
-    # Make sure the logger takes the new environment variables. This is
-    # necessary because the logger is initialized before the environment
-    # variables are set, such as SKYPILOT_DEBUG.
-    sky_logging.reload_logger()
-
 
 def clear_local_api_server_database() -> None:
     """Removes the local API server database.

sky/server/constants.py

@@ -1,11 +1,13 @@
 """Constants for the API servers."""
 
+import os
+
 from sky.skylet import constants
 
 # API server version, whenever there is a change in API server that requires a
 # restart of the local API server or error out when the client does not match
 # the server version.
-API_VERSION = '3'
+API_VERSION = '4'
 
 # Prefix for API request names.
 REQUEST_NAME_PREFIX = 'sky.'
@@ -24,3 +26,7 @@ CLUSTER_REFRESH_DAEMON_INTERVAL_SECONDS = 60
 
 # Environment variable for a file path to the API cookie file.
 API_COOKIE_FILE_ENV_VAR = f'{constants.SKYPILOT_ENV_VAR_PREFIX}API_COOKIE_FILE'
+
+# The path to the dashboard build output
+DASHBOARD_DIR = os.path.join(os.path.dirname(__file__), '..', 'dashboard',
+                             'out')

sky/server/requests/executor.py

@@ -358,6 +358,10 @@ def _request_execution_wrapper(request_id: str,
         # captured in the log file.
         try:
             with override_request_env_and_config(request_body):
+                if sky_logging.logging_enabled(logger, sky_logging.DEBUG):
+                    config = skypilot_config.to_dict()
+                    logger.debug(f'request config: \n'
+                                 f'{common_utils.dump_yaml_str(dict(config))}')
                 return_value = func(**request_body.to_kwargs())
                 f.flush()
         except KeyboardInterrupt: