skypilot-nightly 1.0.0.dev20250413__py3-none-any.whl → 1.0.0.dev20250417__py3-none-any.whl

sky/server/common.py

@@ -17,6 +17,7 @@ import uuid
 import colorama
 import filelock
 
+import sky
 from sky import exceptions
 from sky import sky_logging
 from sky import skypilot_config
@@ -57,12 +58,36 @@ RETRY_COUNT_ON_TIMEOUT = 3
 # (e.g. in high contention env) and we will exit eagerly if server exit.
 WAIT_APISERVER_START_TIMEOUT_SEC = 60
 
-SKY_API_VERSION_WARNING = (
-    f'{colorama.Fore.YELLOW}SkyPilot API server is too old: '
-    f'v{{server_version}} (client version is v{{client_version}}). '
-    'Please restart the SkyPilot API server with: '
+_VERSION_INFO = (
+    f'{colorama.Style.RESET_ALL}'
+    f'{colorama.Style.DIM}'
+    'client version: v{client_version} (API version: v{client_api_version})\n'
+    'server version: v{server_version} (API version: v{server_api_version})'
+    f'{colorama.Style.RESET_ALL}')
+_LOCAL_SERVER_VERSION_MISMATCH_WARNING = (
+    f'{colorama.Fore.YELLOW}Client and local API server version mismatch:\n'
+    '{version_info}\n'
+    f'{colorama.Fore.YELLOW}Please restart the SkyPilot API server with:\n'
     'sky api stop; sky api start'
     f'{colorama.Style.RESET_ALL}')
+_CLIENT_TOO_OLD_WARNING = (
+    f'{colorama.Fore.YELLOW}Your SkyPilot client is too old:\n'
+    '{version_info}\n'
+    f'{colorama.Fore.YELLOW}Upgrade your client with:\n'
+    '{command}'
+    f'{colorama.Style.RESET_ALL}')
+_REMOTE_SERVER_TOO_OLD_WARNING = (
+    f'{colorama.Fore.YELLOW}SkyPilot API server is too old:\n'
+    '{version_info}\n'
+    f'{colorama.Fore.YELLOW}Contact your administrator to upgrade the '
+    'remote API server or downgrade your local client with:\n'
+    '{command}\n'
+    f'{colorama.Style.RESET_ALL}')
+# Parse local API version eargly to catch version format errors.
+_LOCAL_API_VERSION: int = int(server_constants.API_VERSION)
+# SkyPilot dev version.
+_DEV_VERSION = '1.0.0-dev0'
+
 RequestId = str
 ApiVersion = Optional[str]
 
@@ -78,7 +103,9 @@ class ApiServerStatus(enum.Enum):
 @dataclasses.dataclass
 class ApiServerInfo:
     status: ApiServerStatus
-    api_version: ApiVersion
+    api_version: ApiVersion = None
+    version: Optional[str] = None
+    commit: Optional[str] = None
 
 
 def get_api_cookie_jar() -> requests.cookies.RequestsCookieJar:
@@ -137,37 +164,35 @@ def get_api_server_status(endpoint: Optional[str] = None) -> ApiServerInfo:
                 try:
                     result = response.json()
                     api_version = result.get('api_version')
-                    if api_version is None:
+                    version = result.get('version')
+                    commit = result.get('commit')
+                    server_info = ApiServerInfo(status=ApiServerStatus.HEALTHY,
+                                                api_version=api_version,
+                                                version=version,
+                                                commit=commit)
+                    if api_version is None or version is None or commit is None:
                         logger.warning(f'API server response missing '
                                        f'version info. {server_url} may '
                                        f'not be running SkyPilot API server.')
-                        return ApiServerInfo(status=ApiServerStatus.UNHEALTHY,
-                                             api_version=None)
-                    if api_version == server_constants.API_VERSION:
-                        return ApiServerInfo(status=ApiServerStatus.HEALTHY,
-                                             api_version=api_version)
-                    return ApiServerInfo(
-                        status=ApiServerStatus.VERSION_MISMATCH,
-                        api_version=api_version)
+                        server_info.status = ApiServerStatus.UNHEALTHY
+                    elif api_version != server_constants.API_VERSION:
+                        server_info.status = ApiServerStatus.VERSION_MISMATCH
+                    return server_info
                 except (json.JSONDecodeError, AttributeError) as e:
                     logger.warning('Failed to parse API server response: '
                                    f'{str(e)}')
-                    return ApiServerInfo(status=ApiServerStatus.UNHEALTHY,
-                                         api_version=None)
+                    return ApiServerInfo(status=ApiServerStatus.UNHEALTHY)
             else:
-                return ApiServerInfo(status=ApiServerStatus.UNHEALTHY,
-                                     api_version=None)
+                return ApiServerInfo(status=ApiServerStatus.UNHEALTHY)
         except requests.exceptions.Timeout:
             if time_out_try_count == RETRY_COUNT_ON_TIMEOUT:
-                return ApiServerInfo(status=ApiServerStatus.UNHEALTHY,
-                                     api_version=None)
+                return ApiServerInfo(status=ApiServerStatus.UNHEALTHY)
             time_out_try_count += 1
             continue
         except requests.exceptions.ConnectionError:
-            return ApiServerInfo(status=ApiServerStatus.UNHEALTHY,
-                                 api_version=None)
+            return ApiServerInfo(status=ApiServerStatus.UNHEALTHY)
 
-    return ApiServerInfo(status=ApiServerStatus.UNHEALTHY, api_version=None)
+    return ApiServerInfo(status=ApiServerStatus.UNHEALTHY)
 
 
 def handle_request_error(response: 'requests.Response') -> None:
@@ -227,6 +252,7 @@ def _start_api_server(deploy: bool = False,
 
         if foreground:
             # Replaces the current process with the API server
+            os.environ[constants.ENV_VAR_IS_SKYPILOT_SERVER] = 'true'
             os.execvp(args[0], args)
 
         log_path = os.path.expanduser(constants.API_SERVER_LOGS)
@@ -237,7 +263,12 @@ def _start_api_server(deploy: bool = False,
         # If this is called from a CLI invocation, we need
         # start_new_session=True so that SIGINT on the CLI will not also kill
         # the API server.
-        proc = subprocess.Popen(cmd, shell=True, start_new_session=True)
+        server_env = os.environ.copy()
+        server_env[constants.ENV_VAR_IS_SKYPILOT_SERVER] = 'true'
+        proc = subprocess.Popen(cmd,
+                                shell=True,
+                                start_new_session=True,
+                                env=server_env)
 
         start_time = time.time()
         while True:
@@ -247,20 +278,21 @@ def _start_api_server(deploy: bool = False,
                     raise RuntimeError(
                         'SkyPilot API server process exited unexpectedly.\n'
                         f'View logs at: {constants.API_SERVER_LOGS}')
-            api_server_info = get_api_server_status()
-            assert api_server_info.status != ApiServerStatus.VERSION_MISMATCH, (
-                f'API server version mismatch when starting the server. '
-                f'Server version: {api_server_info.api_version} '
-                f'Client version: {server_constants.API_VERSION}')
-            if api_server_info.status == ApiServerStatus.HEALTHY:
+            try:
+                check_server_healthy()
+            except exceptions.APIVersionMismatchError:
+                raise
+            except Exception as e:  # pylint: disable=broad-except
+                if time.time() - start_time >= WAIT_APISERVER_START_TIMEOUT_SEC:
+                    with ux_utils.print_exception_no_traceback():
+                        raise RuntimeError(
+                            'Failed to start SkyPilot API server at '
+                            f'{get_server_url(host)}'
+                            '\nView logs at: '
+                            f'{constants.API_SERVER_LOGS}') from e
+                time.sleep(0.5)
+            else:
                 break
-            elif time.time() - start_time >= WAIT_APISERVER_START_TIMEOUT_SEC:
-                with ux_utils.print_exception_no_traceback():
-                    raise RuntimeError(
-                        'Failed to start SkyPilot API server at '
-                        f'{get_server_url(host)}'
-                        f'\nView logs at: {constants.API_SERVER_LOGS}')
-            time.sleep(0.5)
         logger.info(ux_utils.finishing_message('SkyPilot API server started.'))
 
 
@@ -279,16 +311,70 @@ def check_server_healthy(endpoint: Optional[str] = None,) -> None:
     api_server_info = get_api_server_status(endpoint)
     api_server_status = api_server_info.status
     if api_server_status == ApiServerStatus.VERSION_MISMATCH:
+        sv = api_server_info.api_version
+        assert sv is not None, 'Server API version is None'
+        try:
+            server_is_older = int(sv) < _LOCAL_API_VERSION
+        except ValueError:
+            # Raised when the server version using an unknown scheme.
+            # Version compatibility checking is expected to handle all legacy
+            # cases so we safely assume the server is newer when the version
+            # scheme is unknown.
+            logger.debug('API server version using unknown scheme: %s', sv)
+            server_is_older = False
+        version_info = _get_version_info_hint(api_server_info)
+        if is_api_server_local():
+            # For local server, just hint user to restart the server to get
+            # a consistent version.
+            msg = _LOCAL_SERVER_VERSION_MISMATCH_WARNING.format(
+                version_info=version_info)
+        else:
+            assert api_server_info.version is not None, 'Server version is None'
+            if server_is_older:
+                msg = _REMOTE_SERVER_TOO_OLD_WARNING.format(
+                    version_info=version_info,
+                    command=_install_server_version_command(api_server_info))
+            else:
+                msg = _CLIENT_TOO_OLD_WARNING.format(
+                    version_info=version_info,
+                    command=_install_server_version_command(api_server_info))
         with ux_utils.print_exception_no_traceback():
-            raise RuntimeError(
-                SKY_API_VERSION_WARNING.format(
-                    server_version=api_server_info.api_version,
-                    client_version=server_constants.API_VERSION))
+            raise exceptions.APIVersionMismatchError(msg)
     elif api_server_status == ApiServerStatus.UNHEALTHY:
         with ux_utils.print_exception_no_traceback():
             raise exceptions.ApiServerConnectionError(endpoint)
 
 
+def _get_version_info_hint(server_info: ApiServerInfo) -> str:
+    assert server_info.version is not None, 'Server version is None'
+    assert server_info.commit is not None, 'Server commit is None'
+    sv = server_info.version
+    cv = sky.__version__
+    if server_info.version == _DEV_VERSION:
+        sv = f'{sv} with commit {server_info.commit}'
+    if cv == _DEV_VERSION:
+        cv = f'{cv} with commit {sky.__commit__}'
+    return _VERSION_INFO.format(client_version=cv,
+                                server_version=sv,
+                                client_api_version=server_constants.API_VERSION,
+                                server_api_version=server_info.api_version)
+
+
+def _install_server_version_command(server_info: ApiServerInfo) -> str:
+    assert server_info.version is not None, 'Server version is None'
+    assert server_info.commit is not None, 'Server commit is None'
+    if server_info.version == _DEV_VERSION:
+        # Dev build without valid version.
+        return ('pip install git+https://github.com/skypilot-org/skypilot@'
+                f'{server_info.commit}')
+    elif 'dev' in server_info.version:
+        # Nightly version.
+        return f'pip install -U "skypilot-nightly=={server_info.version}"'
+    else:
+        # Stable version.
+        return f'pip install -U "skypilot=={server_info.version}"'
+
+
 def check_server_healthy_or_start_fn(deploy: bool = False,
                                      host: str = '127.0.0.1',
                                      foreground: bool = False):
@@ -436,6 +522,12 @@ def reload_for_new_request(client_entrypoint: Optional[str],
                            client_command: Optional[str],
                            using_remote_api_server: bool):
     """Reload modules, global variables, and usage message for a new request."""
+    # This should be called first to make sure the logger is up-to-date.
+    sky_logging.reload_logger()
+
+    # Reload the skypilot config to make sure the latest config is used.
+    skypilot_config.safe_reload_config()
+
     # Reset the client entrypoint and command for the usage message.
     common_utils.set_client_status(
         client_entrypoint=client_entrypoint,
@@ -452,11 +544,6 @@ def reload_for_new_request(client_entrypoint: Optional[str],
     # latest information in the context, e.g. client entrypoint and run id.
     usage_lib.messages.reset(usage_lib.MessageType.USAGE)
 
-    # Make sure the logger takes the new environment variables. This is
-    # necessary because the logger is initialized before the environment
-    # variables are set, such as SKYPILOT_DEBUG.
-    sky_logging.reload_logger()
-
 
 def clear_local_api_server_database() -> None:
     """Removes the local API server database.

sky/server/constants.py

@@ -5,7 +5,7 @@ from sky.skylet import constants
 # API server version, whenever there is a change in API server that requires a
 # restart of the local API server or error out when the client does not match
 # the server version.
-API_VERSION = '3'
+API_VERSION = '4'
 
 # Prefix for API request names.
 REQUEST_NAME_PREFIX = 'sky.'

sky/server/requests/serializers/decoders.py

@@ -188,8 +188,5 @@ def decode_job_status(
 
 @register_decoders('kubernetes_node_info')
 def decode_kubernetes_node_info(
-        return_value: Dict[str, Any]) -> Dict[str, models.KubernetesNodeInfo]:
-    return {
-        node_name: models.KubernetesNodeInfo(**node_info)
-        for node_name, node_info in return_value.items()
-    }
+        return_value: Dict[str, Any]) -> models.KubernetesNodesInfo:
+    return models.KubernetesNodesInfo.from_dict(return_value)

sky/server/requests/serializers/encoders.py

@@ -159,8 +159,5 @@ def encode_job_status(return_value: Dict[int, Any]) -> Dict[int, str]:
 
 @register_encoder('kubernetes_node_info')
 def encode_kubernetes_node_info(
-        return_value: Dict[str, 'models.KubernetesNodeInfo']) -> Dict[str, Any]:
-    return {
-        node_name: dataclasses.asdict(node_info)
-        for node_name, node_info in return_value.items()
-    }
+        return_value: 'models.KubernetesNodesInfo') -> Dict[str, Any]:
+    return return_value.to_dict()

sky/server/server.py

@@ -210,7 +210,7 @@ async def kubernetes_node_info(
         request: fastapi.Request,
         kubernetes_node_info_body: payloads.KubernetesNodeInfoRequestBody
 ) -> None:
-    """Gets Kubernetes node information."""
+    """Gets Kubernetes nodes information and hints."""
     executor.schedule_request(
         request_id=request.state.request_id,
         request_name='kubernetes_node_info',

sky/setup_files/dependencies.py

@@ -53,6 +53,7 @@ install_requires = [
     'aiofiles',
     'httpx',
     'setproctitle',
+    'omegaconf>=2.4.0dev3,<2.5',
 ]
 
 local_ray = [

sky/sky_logging.py

@@ -97,8 +97,8 @@ def _setup_logger():
 def reload_logger():
     """Reload the logger.
 
-    This is useful when the logging configuration is changed.
-    e.g., the logging level is changed or stdout/stderr is reset.
+    This ensures that the logger takes the new environment variables,
+    such as SKYPILOT_DEBUG.
     """
     global _default_handler
     _root_logger.removeHandler(_default_handler)

sky/skylet/constants.py

@@ -117,7 +117,7 @@ RUNPOD_DOCKER_USERNAME_ENV_VAR = 'SKYPILOT_RUNPOD_DOCKER_USERNAME'
 
 # Commands for disable GPU ECC, which can improve the performance of the GPU
 # for some workloads by 30%. This will only be applied when a user specify
-# `nvidia_gpus.disable_ecc: true` in ~/.sky/skyconfig.yaml.
+# `nvidia_gpus.disable_ecc: true` in ~/.sky/config.yaml.
 # Running this command will reboot the machine, introducing overhead for
 # provisioning the machine.
 # https://portal.nutanix.com/page/documents/kbs/details?targetId=kA00e000000LKjOCAW
@@ -299,11 +299,6 @@ FILE_MOUNTS_WORKDIR_SUBPATH = 'job-{run_id}/workdir'
 FILE_MOUNTS_SUBPATH = 'job-{run_id}/local-file-mounts/{i}'
 FILE_MOUNTS_TMP_SUBPATH = 'job-{run_id}/tmp-files'
 
-# The default idle timeout for SkyPilot controllers. This include jobs
-# controller and sky serve controller.
-# TODO(tian): Refactor to controller_utils. Current blocker: circular import.
-CONTROLLER_IDLE_MINUTES_TO_AUTOSTOP = 10
-
 # Due to the CPU/memory usage of the controller process launched with sky jobs (
 # use ray job under the hood), we need to reserve some CPU/memory for each jobs/
 # serve controller process.
@@ -337,7 +332,7 @@ RCLONE_LOG_DIR = '~/.sky/rclone_log'
 RCLONE_CACHE_DIR = '~/.cache/rclone'
 RCLONE_CACHE_REFRESH_INTERVAL = 10
 
-# The keys that can be overridden in the `~/.sky/skyconfig.yaml` file. The
+# The keys that can be overridden in the `~/.sky/config.yaml` file. The
 # overrides are specified in task YAMLs.
 OVERRIDEABLE_CONFIG_KEYS_IN_TASK: List[Tuple[str, ...]] = [
     ('docker', 'run_options'),
@@ -367,3 +362,6 @@ LOCAL_SKYPILOT_CONFIG_PATH_PLACEHOLDER = 'skypilot:local_skypilot_config_path'
 
 # Path to the generated cluster config yamls and ssh configs.
 SKY_USER_FILE_PATH = '~/.sky/generated'
+
+# Environment variable that is set to 'true' if this is a skypilot server.
+ENV_VAR_IS_SKYPILOT_SERVER = 'IS_SKYPILOT_SERVER'

sky/skylet/job_lib.py

@@ -630,7 +630,7 @@ def update_job_status(job_ids: List[int],
                         'it to FAILED_DRIVER')
                     status = JobStatus.FAILED_DRIVER
             elif job_pid < 0:
-                # TODO(zhwu): Backward compatibility, remove after 0.9.0.
+                # TODO(zhwu): Backward compatibility, remove after 0.10.0.
                 # We set the job status to PENDING instead of actually
                 # checking ray job status and let the status in job table
                 # take effect in the later max.
@@ -882,7 +882,7 @@ def cancel_jobs_encoded_results(jobs: Optional[List[int]],
                 # child processes.
             elif job['pid'] < 0:
                 try:
-                    # TODO(zhwu): Backward compatibility, remove after 0.9.0.
+                    # TODO(zhwu): Backward compatibility, remove after 0.10.0.
                     # The job was submitted with ray job submit before #4318.
                     job_client = _create_ray_job_submission_client()
                     job_client.stop_job(_make_ray_job_id(job['job_id']))
@@ -1008,7 +1008,7 @@ class JobLibCodeGen:
             # Print cancelled IDs. Caller should parse by decoding.
             'print(cancelled, flush=True)',
         ]
-        # TODO(zhwu): Backward compatibility, remove after 0.9.0.
+        # TODO(zhwu): Backward compatibility, remove after 0.12.0.
         if user_hash is None:
             code = [
                 (f'cancelled = job_lib.cancel_jobs_encoded_results('