skypilot-nightly 1.0.0.dev20250521__py3-none-any.whl → 1.0.0.dev20250523__py3-none-any.whl

sky/provision/kubernetes/utils.py

@@ -1137,6 +1137,11 @@ def get_accelerator_label_key_values(
     #  support pollingthe clusters for autoscaling information, such as the
     #  node pools configured etc.
 
+    is_ssh_node_pool = context.startswith('ssh-') if context else False
+    cloud_name = 'SSH Node Pool' if is_ssh_node_pool else 'Kubernetes cluster'
+    context_display_name = context.lstrip('ssh-') if (
+        context and is_ssh_node_pool) else context
+
     autoscaler_type = get_autoscaler_type()
     if autoscaler_type is not None:
         # If autoscaler is set in config.yaml, override the label key and value
@@ -1176,13 +1181,17 @@ def get_accelerator_label_key_values(
                 suffix = ''
                 if env_options.Options.SHOW_DEBUG_INFO.get():
                     suffix = f' Found node labels: {node_labels}'
-                raise exceptions.ResourcesUnavailableError(
-                    'Could not detect GPU labels in Kubernetes cluster. '
-                    'If this cluster has GPUs, please ensure GPU nodes have '
-                    'node labels of either of these formats: '
-                    f'{supported_formats}. Please refer to '
-                    'the documentation on how to set up node labels.'
-                    f'{suffix}')
+                msg = (f'Could not detect GPU labels in {cloud_name}.')
+                if not is_ssh_node_pool:
+                    msg += (' Run `sky check ssh` to debug.')
+                else:
+                    msg += (
+                        ' If this cluster has GPUs, please ensure GPU nodes have '
+                        'node labels of either of these formats: '
+                        f'{supported_formats}. Please refer to '
+                        'the documentation on how to set up node labels.')
+                msg += f'{suffix}'
+                raise exceptions.ResourcesUnavailableError(msg)
         else:
             # Validate the label value on all nodes labels to ensure they are
             # correctly setup and will behave as expected.
@@ -1193,7 +1202,7 @@ def get_accelerator_label_key_values(
                             value)
                         if not is_valid:
                             raise exceptions.ResourcesUnavailableError(
-                                f'Node {node_name!r} in Kubernetes cluster has '
+                                f'Node {node_name!r} in {cloud_name} has '
                                 f'invalid GPU label: {label}={value}. {reason}')
             if check_mode:
                 # If check mode is enabled and we reached so far, we can
@@ -1257,10 +1266,10 @@ def get_accelerator_label_key_values(
                 # TODO(Doyoung): Update the error message raised with the
                 # multi-host TPU support.
                 raise exceptions.ResourcesUnavailableError(
-                    'Could not find any node in the Kubernetes cluster '
+                    f'Could not find any node in the {cloud_name} '
                     f'with {acc_type}. Please ensure at least one node in the '
                     f'cluster has {acc_type} and node labels are setup '
-                    'correctly. Please refer to the documentration for more. '
+                    'correctly. Please refer to the documentation for more. '
                     f'{suffix}. Note that multi-host TPU podslices are '
                     'currently not unsupported.')
     else:
@@ -1270,15 +1279,24 @@ def get_accelerator_label_key_values(
             if env_options.Options.SHOW_DEBUG_INFO.get():
                 suffix = (' Available resources on the cluster: '
                           f'{cluster_resources}')
-            raise exceptions.ResourcesUnavailableError(
-                f'Could not detect GPU/TPU resources ({GPU_RESOURCE_KEY!r} or '
-                f'{TPU_RESOURCE_KEY!r}) in Kubernetes cluster. If this cluster'
-                ' contains GPUs, please ensure GPU drivers are installed on '
-                'the node. Check if the GPUs are setup correctly by running '
-                '`kubectl describe nodes` and looking for the '
-                f'{GPU_RESOURCE_KEY!r} or {TPU_RESOURCE_KEY!r} resource. '
-                'Please refer to the documentation on how to set up GPUs.'
-                f'{suffix}')
+            if is_ssh_node_pool:
+                msg = (
+                    f'Could not detect GPUs in SSH Node Pool '
+                    f'\'{context_display_name}\'. If this cluster contains '
+                    'GPUs, please ensure GPU drivers are installed on the node '
+                    'and re-run '
+                    f'`sky ssh up --infra {context_display_name}`. {suffix}')
+            else:
+                msg = (
+                    f'Could not detect GPU/TPU resources ({GPU_RESOURCE_KEY!r} or '
+                    f'{TPU_RESOURCE_KEY!r}) in Kubernetes cluster. If this cluster'
+                    ' contains GPUs, please ensure GPU drivers are installed on '
+                    'the node. Check if the GPUs are setup correctly by running '
+                    '`kubectl describe nodes` and looking for the '
+                    f'{GPU_RESOURCE_KEY!r} or {TPU_RESOURCE_KEY!r} resource. '
+                    'Please refer to the documentation on how to set up GPUs.'
+                    f'{suffix}')
+            raise exceptions.ResourcesUnavailableError(msg)
     assert False, 'This should not be reached'
 
 

sky/provision/nebius/instance.py

@@ -134,7 +134,9 @@ def run_instances(region: str, cluster_name_on_cloud: str,
                 disk_size=config.node_config['DiskSize'],
                 user_data=config.node_config['UserData'],
                 associate_public_ip_address=(
-                    not config.provider_config['use_internal_ips']))
+                    not config.provider_config['use_internal_ips']),
+                filesystems=config.node_config.get('filesystems', []),
+            )
         except Exception as e:  # pylint: disable=broad-except
             logger.warning(f'run_instances error: {e}')
             raise

sky/provision/nebius/utils.py

@@ -1,6 +1,6 @@
 """Nebius library wrapper for SkyPilot."""
 import time
-from typing import Any, Dict
+from typing import Any, Dict, List
 import uuid
 
 from sky import sky_logging
@@ -158,7 +158,8 @@ def start(instance_id: str) -> None:
 
 def launch(cluster_name_on_cloud: str, node_type: str, platform: str,
            preset: str, region: str, image_family: str, disk_size: int,
-           user_data: str, associate_public_ip_address: bool) -> str:
+           user_data: str, associate_public_ip_address: bool,
+           filesystems: List[Dict[str, Any]]) -> str:
     # Each node must have a unique name to avoid conflicts between
     # multiple worker VMs. To ensure uniqueness,a UUID is appended
     # to the node name.
@@ -217,6 +218,16 @@ def launch(cluster_name_on_cloud: str, node_type: str, platform: str,
             f' seconds) while waiting for disk {disk_name}'
             f' to be ready.')
 
+    filesystems_spec = []
+    if filesystems:
+        for fs in filesystems:
+            filesystems_spec.append(nebius.compute().AttachedFilesystemSpec(
+                mount_tag=fs['filesystem_mount_tag'],
+                attach_mode=nebius.compute().AttachedFilesystemSpec.AttachMode[
+                    fs['filesystem_attach_mode']],
+                existing_filesystem=nebius.compute().ExistingFilesystem(
+                    id=fs['filesystem_id'])))
+
     service = nebius.vpc().SubnetServiceClient(nebius.sdk())
     sub_net = service.list(nebius.vpc().ListSubnetsRequest(
         parent_id=project_id,)).wait()
@@ -237,6 +248,7 @@ def launch(cluster_name_on_cloud: str, node_type: str, platform: str,
             cloud_init_user_data=user_data,
             resources=nebius.compute().ResourcesSpec(platform=platform,
                                                      preset=preset),
+            filesystems=filesystems_spec if filesystems_spec else None,
             network_interfaces=[
                 nebius.compute().NetworkInterfaceSpec(
                     subnet_id=sub_net.items[0].metadata.id,

sky/provision/ssh/__init__.py

@@ -0,0 +1,18 @@
+"""SSH provisioner for SkyPilot.
+
+This module implements the provisioner interface for SSH targets.
+It reuses most of the functionality from the Kubernetes provisioner,
+since the SSH implementation is based on Kubernetes under the hood.
+"""
+
+from sky.provision.kubernetes.config import bootstrap_instances
+from sky.provision.kubernetes.instance import get_cluster_info
+from sky.provision.kubernetes.instance import get_command_runners
+from sky.provision.kubernetes.instance import query_instances
+from sky.provision.kubernetes.instance import run_instances
+from sky.provision.kubernetes.instance import stop_instances
+from sky.provision.kubernetes.instance import terminate_instances
+from sky.provision.kubernetes.instance import wait_instances
+from sky.provision.kubernetes.network import cleanup_ports
+from sky.provision.kubernetes.network import open_ports
+from sky.provision.kubernetes.network import query_ports

sky/resources.py

@@ -98,7 +98,7 @@ class Resources:
     """
     # If any fields changed, increment the version. For backward compatibility,
     # modify the __setstate__ method to handle the old version.
-    _VERSION = 23
+    _VERSION = 24
 
     def __init__(
         self,
@@ -120,6 +120,7 @@ class Resources:
         ports: Optional[Union[int, str, List[str], Tuple[str]]] = None,
         labels: Optional[Dict[str, str]] = None,
         autostop: Union[bool, int, Dict[str, Any], None] = None,
+        volumes: Optional[List[Dict[str, Any]]] = None,
         # Internal use only.
         # pylint: disable=invalid-name
         _docker_login_config: Optional[docker_utils.DockerLoginConfig] = None,
@@ -210,6 +211,7 @@ class Resources:
             not supported and will be ignored.
           autostop: the autostop configuration to use. For launched resources,
             may or may not correspond to the actual current autostop config.
+          volumes: the volumes to mount on the instance.
           _docker_login_config: the docker configuration to use. This includes
             the docker username, password, and registry server. If None, skip
             docker login.
@@ -337,6 +339,7 @@ class Resources:
         self._set_memory(memory)
         self._set_accelerators(accelerators, accelerator_args)
         self._set_autostop_config(autostop)
+        self._set_volumes(volumes)
 
     def validate(self):
         """Validate the resources and infer the missing fields if possible."""
@@ -347,6 +350,7 @@ class Resources:
         self._try_validate_managed_job_attributes()
         self._try_validate_image_id()
         self._try_validate_disk_tier()
+        self._try_validate_volumes()
         self._try_validate_ports()
         self._try_validate_labels()
 
@@ -448,7 +452,10 @@ class Resources:
     def repr_with_region_zone(self) -> str:
         region_str = ''
         if self.region is not None:
-            region_str = f', region={self.region}'
+            region_name = self.region
+            if self.region.startswith('ssh-'):
+                region_name = self.region.lstrip('ssh-')
+            region_str = f', region={region_name}'
         zone_str = ''
         if self.zone is not None:
             zone_str = f', zone={self.zone}'
@@ -566,6 +573,10 @@ class Resources:
     def labels(self) -> Optional[Dict[str, str]]:
         return self._labels
 
+    @property
+    def volumes(self) -> Optional[List[Dict[str, Any]]]:
+        return self._volumes
+
     @property
     def autostop_config(self) -> Optional[AutostopConfig]:
         """The requested autostop config.
@@ -759,6 +770,91 @@ class Resources:
     ) -> None:
         self._autostop_config = AutostopConfig.from_yaml_config(autostop)
 
+    def _set_volumes(
+        self,
+        volumes: Optional[List[Dict[str, Any]]],
+    ) -> None:
+        if not volumes:
+            self._volumes = None
+            return
+        valid_volumes = []
+        supported_tiers = [tier.value for tier in resources_utils.DiskTier]
+        supported_storage_types = [
+            storage_type.value for storage_type in resources_utils.StorageType
+        ]
+        supported_attach_modes = [
+            attach_mode.value for attach_mode in resources_utils.DiskAttachMode
+        ]
+        network_type = resources_utils.StorageType.NETWORK
+        read_write_mode = resources_utils.DiskAttachMode.READ_WRITE
+        for volume in volumes:
+            if 'path' not in volume:
+                with ux_utils.print_exception_no_traceback():
+                    raise ValueError(f'Invalid volume {volume!r}. '
+                                     f'Volume must have a "path" field.')
+            if 'storage_type' not in volume:
+                volume['storage_type'] = network_type
+            else:
+                if isinstance(volume['storage_type'], str):
+                    storage_type_str = str(volume['storage_type']).lower()
+                    if storage_type_str not in supported_storage_types:
+                        logger.warning(
+                            f'Invalid storage_type {storage_type_str!r}. '
+                            f'Set it to '
+                            f'{network_type.value}.')
+                        volume['storage_type'] = network_type
+                    else:
+                        volume['storage_type'] = resources_utils.StorageType(
+                            storage_type_str)
+            if 'auto_delete' not in volume:
+                volume['auto_delete'] = False
+            if 'attach_mode' in volume:
+                if isinstance(volume['attach_mode'], str):
+                    attach_mode_str = str(volume['attach_mode']).lower()
+                    if attach_mode_str not in supported_attach_modes:
+                        logger.warning(
+                            f'Invalid attach_mode {attach_mode_str!r}. '
+                            f'Set it to {read_write_mode.value}.')
+                        volume['attach_mode'] = read_write_mode
+                    else:
+                        volume['attach_mode'] = resources_utils.DiskAttachMode(
+                            attach_mode_str)
+            else:
+                volume['attach_mode'] = read_write_mode
+            if volume['storage_type'] == network_type:
+                if ('disk_size' in volume and
+                        round(volume['disk_size']) != volume['disk_size']):
+                    with ux_utils.print_exception_no_traceback():
+                        raise ValueError(f'Volume size must be an integer. '
+                                         f'Got: {volume["size"]}.')
+                if 'name' not in volume:
+                    with ux_utils.print_exception_no_traceback():
+                        raise ValueError(f'Network volume {volume["path"]} '
+                                         f'must have "name" field.')
+            elif 'name' in volume:
+                logger.info(f'Volume {volume["path"]} is a local disk. '
+                            f'The "name" field will be ignored.')
+                del volume['name']
+            if 'disk_tier' in volume:
+                if isinstance(volume['disk_tier'], str):
+                    disk_tier_str = str(volume['disk_tier']).lower()
+                    if disk_tier_str not in supported_tiers:
+                        logger.warning(
+                            f'Invalid disk_tier {disk_tier_str!r}. '
+                            f'Set it to {resources_utils.DiskTier.BEST.value}.')
+                        volume['disk_tier'] = resources_utils.DiskTier.BEST
+                    else:
+                        volume['disk_tier'] = resources_utils.DiskTier(
+                            disk_tier_str)
+            elif volume['storage_type'] == network_type:
+                logger.debug(
+                    f'No disk_tier specified for volume {volume["path"]}. '
+                    f'Set it to {resources_utils.DiskTier.BEST.value}.')
+                volume['disk_tier'] = resources_utils.DiskTier.BEST
+
+            valid_volumes.append(volume)
+        self._volumes = valid_volumes
+
     def is_launchable(self) -> bool:
         """Returns whether the resource is launchable."""
         return self.cloud is not None and self._instance_type is not None
@@ -1123,6 +1219,48 @@ class Resources:
                         f'Disk tier {self.disk_tier.value} is not supported '
                         f'for instance type {self.instance_type}.') from None
 
+    def _try_validate_volumes(self) -> None:
+        """Try to validate the volumes attribute.
+
+        Raises:
+            ValueError: if the attribute is invalid.
+        """
+        if self.volumes is None:
+            return
+        if self.cloud is None:
+            with ux_utils.print_exception_no_traceback():
+                raise ValueError('Cloud must be specified when '
+                                 'volumes are provided.')
+        if not self.cloud.is_same_cloud(clouds.GCP()):
+            with ux_utils.print_exception_no_traceback():
+                raise ValueError(f'Volumes are only supported for GCP'
+                                 f' not for {self.cloud}.')
+
+        need_region_or_zone = False
+        try:
+            for volume in self.volumes:
+                if ('name' in volume and volume['storage_type']
+                        == resources_utils.StorageType.NETWORK):
+                    need_region_or_zone = True
+                if 'disk_tier' not in volume:
+                    continue
+                # TODO(hailong): check instance local SSD
+                # support for instance_type.
+                # Refer to https://cloud.google.com/compute/docs/disks/local-ssd#machine-series-lssd # pylint: disable=line-too-long
+                self.cloud.check_disk_tier_enabled(self.instance_type,
+                                                   volume['disk_tier'])
+            if (need_region_or_zone and self._region is None and
+                    self._zone is None):
+                with ux_utils.print_exception_no_traceback():
+                    raise ValueError('When specifying the volume name, please'
+                                     ' also specify the region or zone.')
+        except exceptions.NotSupportedError:
+            with ux_utils.print_exception_no_traceback():
+                raise ValueError(
+                    f'Disk tier {volume["disk_tier"].value} is not '
+                    f'supported for instance type {self.instance_type}.'
+                ) from None
+
     def _try_validate_ports(self) -> None:
         """Try to validate the ports attribute.
 
@@ -1293,9 +1431,18 @@ class Resources:
             skypilot_config.get_nested(
                 (str(self.cloud).lower(), 'specific_reservations'), set()))
 
+        if isinstance(self.cloud, clouds.DummyCloud):
+            return self.cloud.get_reservations_available_resources(
+                instance_type='',
+                region='',
+                zone=None,
+                specific_reservations=specific_reservations)
+
         assert (self.cloud is not None and self.instance_type is not None and
-                self.region
-                is not None), ('Cloud, instance type, region must be specified')
+                self.region is not None), (
+                    f'Cloud, instance type, region must be specified. '
+                    f'Resources={self}, cloud={self.cloud}, '
+                    f'instance_type={self.instance_type}, region={self.region}')
         return self.cloud.get_reservations_available_resources(
             self.instance_type, self.region, self.zone, specific_reservations)
 
@@ -1483,6 +1630,7 @@ class Resources:
             ports=override.pop('ports', self.ports),
             labels=override.pop('labels', self.labels),
             autostop=override.pop('autostop', current_autostop_config),
+            volumes=override.pop('volumes', self.volumes),
             infra=override.pop('infra', None),
             _docker_login_config=override.pop('_docker_login_config',
                                               self._docker_login_config),
@@ -1523,6 +1671,12 @@ class Resources:
             features.add(clouds.CloudImplementationFeatures.IMAGE_ID)
         if self.ports is not None:
             features.add(clouds.CloudImplementationFeatures.OPEN_PORTS)
+        if self.volumes is not None:
+            for volume in self.volumes:
+                if 'disk_tier' in volume and volume[
+                        'disk_tier'] != resources_utils.DiskTier.BEST:
+                    features.add(
+                        clouds.CloudImplementationFeatures.CUSTOM_DISK_TIER)
         return features
 
     @staticmethod
@@ -1692,6 +1846,7 @@ class Resources:
         resources_fields['ports'] = config.pop('ports', None)
         resources_fields['labels'] = config.pop('labels', None)
         resources_fields['autostop'] = config.pop('autostop', None)
+        resources_fields['volumes'] = config.pop('volumes', None)
         resources_fields['_docker_login_config'] = config.pop(
             '_docker_login_config', None)
         resources_fields['_docker_username_for_runpod'] = config.pop(
@@ -1742,6 +1897,21 @@ class Resources:
             config['disk_tier'] = self.disk_tier.value
         add_if_not_none('ports', self.ports)
         add_if_not_none('labels', self.labels)
+        if self.volumes is not None:
+            # Convert DiskTier/StorageType enum to string value for each volume
+            volumes = []
+            for volume in self.volumes:
+                volume_copy = volume.copy()
+                if 'disk_tier' in volume_copy:
+                    volume_copy['disk_tier'] = volume_copy['disk_tier'].value
+                if 'storage_type' in volume_copy:
+                    volume_copy['storage_type'] = volume_copy[
+                        'storage_type'].value
+                if 'attach_mode' in volume_copy:
+                    volume_copy['attach_mode'] = volume_copy[
+                        'attach_mode'].value
+                volumes.append(volume_copy)
+            config['volumes'] = volumes
         if self._autostop_config is not None:
             config['autostop'] = self._autostop_config.to_yaml_config()
         if self._docker_login_config is not None:
@@ -1902,6 +2072,9 @@ class Resources:
         if version < 23:
             self._autostop_config = None
 
+        if version < 24:
+            self._volumes = None
+
         self.__dict__.update(state)
 
 

sky/serve/server/core.py

@@ -141,8 +141,7 @@ def up(
     # Always apply the policy again here, even though it might have been applied
     # in the CLI. This is to ensure that we apply the policy to the final DAG
     # and get the mutated config.
-    dag, mutated_user_config = admin_policy_utils.apply(
-        task, use_mutated_config_in_current_request=False)
+    dag, mutated_user_config = admin_policy_utils.apply(task)
     task = dag.tasks[0]
 
     with rich_utils.safe_status(
@@ -352,8 +351,7 @@ def update(
     # and get the mutated config.
     # TODO(cblmemo,zhwu): If a user sets a new skypilot_config, the update
     # will not apply the config.
-    dag, _ = admin_policy_utils.apply(
-        task, use_mutated_config_in_current_request=False)
+    dag, _ = admin_policy_utils.apply(task)
     task = dag.tasks[0]
 
     assert task.service is not None

sky/server/common.py

@@ -12,7 +12,7 @@ import subprocess
 import sys
 import time
 import typing
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Literal, Optional
 from urllib import parse
 import uuid
 
@@ -116,6 +116,7 @@ class ApiServerStatus(enum.Enum):
     HEALTHY = 'healthy'
     UNHEALTHY = 'unhealthy'
     VERSION_MISMATCH = 'version_mismatch'
+    NEEDS_AUTH = 'needs_auth'
 
 
 @dataclasses.dataclass
@@ -127,15 +128,21 @@ class ApiServerInfo:
     commit: Optional[str] = None
 
 
+def get_api_cookie_jar_path() -> str:
+    return os.environ.get(server_constants.API_COOKIE_FILE_ENV_VAR,
+                          server_constants.API_COOKIE_FILE_DEFAULT_LOCATION)
+
+
 def get_api_cookie_jar() -> requests.cookies.RequestsCookieJar:
     """Returns the cookie jar used by the client to access the API server."""
-    cookie_file = os.environ.get(server_constants.API_COOKIE_FILE_ENV_VAR)
     cookie_jar = requests.cookies.RequestsCookieJar()
-    if cookie_file and os.path.exists(cookie_file):
+    cookie_file = get_api_cookie_jar_path()
+    if cookie_file:
         cookie_path = pathlib.Path(cookie_file).expanduser().resolve()
-        file_cookie_jar = MozillaCookieJar(cookie_path)
-        file_cookie_jar.load()
-        cookie_jar.update(file_cookie_jar)
+        if cookie_path.exists():
+            file_cookie_jar = MozillaCookieJar(cookie_path)
+            file_cookie_jar.load()
+            cookie_jar.update(file_cookie_jar)
     return cookie_jar
 
 
@@ -196,6 +203,7 @@ def get_api_server_status(endpoint: Optional[str] = None) -> ApiServerInfo:
             response = requests.get(f'{server_url}/api/health',
                                     timeout=2.5,
                                     cookies=get_api_cookie_jar())
+            logger.debug(f'Health check status: {response.status_code}')
             if response.status_code == 200:
                 try:
                     result = response.json()
@@ -217,9 +225,24 @@ def get_api_server_status(endpoint: Optional[str] = None) -> ApiServerInfo:
                         server_info.status = ApiServerStatus.VERSION_MISMATCH
                     return server_info
                 except (json.JSONDecodeError, AttributeError) as e:
+                    # Try to check if we got redirected to a login page.
+                    for prev_response in response.history:
+                        logger.debug(f'Previous response: {prev_response.url}')
+                        # Heuristic: check if the url looks like a login page or
+                        # oauth flow.
+                        if any(key in prev_response.url
+                               for key in ['login', 'oauth2']):
+                            logger.debug(
+                                f'URL {prev_response.url} looks like '
+                                'a login page or oauth flow, so try to '
+                                'get the cookie.')
+                            return ApiServerInfo(
+                                status=ApiServerStatus.NEEDS_AUTH)
                     logger.warning('Failed to parse API server response: '
                                    f'{str(e)}')
                     return ApiServerInfo(status=ApiServerStatus.UNHEALTHY)
+            elif response.status_code == 401:
+                return ApiServerInfo(status=ApiServerStatus.NEEDS_AUTH)
             else:
                 return ApiServerInfo(status=ApiServerStatus.UNHEALTHY)
         except requests.exceptions.Timeout:
@@ -369,7 +392,12 @@ def _start_api_server(deploy: bool = False,
                 f'SkyPilot API server started. {dashboard_msg}'))
 
 
-def check_server_healthy(endpoint: Optional[str] = None,) -> None:
+def check_server_healthy(
+    endpoint: Optional[str] = None
+) -> Literal[
+        # Use an incomplete list of Literals here to enforce raising for other
+        # enum values.
+        ApiServerStatus.HEALTHY, ApiServerStatus.NEEDS_AUTH]:
     """Check if the API server is healthy.
 
     Args:
@@ -379,6 +407,11 @@ def check_server_healthy(endpoint: Optional[str] = None,) -> None:
     Raises:
         RuntimeError: If the server is not healthy or the client version does
             not match the server version.
+
+    Returns:
+        ApiServerStatus: The status of the API server, unless the server is
+            unhealthy or the client version does not match the server version,
+            in which case an exception is raised.
     """
     endpoint = endpoint if endpoint is not None else get_server_url()
     api_server_info = get_api_server_status(endpoint)
@@ -441,6 +474,8 @@ def check_server_healthy(endpoint: Optional[str] = None,) -> None:
 
         hinted_for_server_install_version_mismatch = True
 
+    return api_server_status
+
 
 def _get_version_info_hint(server_info: ApiServerInfo) -> str:
     assert server_info.version is not None, 'Server version is None'
@@ -491,11 +526,13 @@ def get_skypilot_version_on_disk() -> str:
 def check_server_healthy_or_start_fn(deploy: bool = False,
                                      host: str = '127.0.0.1',
                                      foreground: bool = False):
+    api_server_status = None
     try:
-        check_server_healthy()
+        api_server_status = check_server_healthy()
     except exceptions.ApiServerConnectionError as exc:
         endpoint = get_server_url()
-        if not is_api_server_local():
+        if (not is_api_server_local() or
+                api_server_status == ApiServerStatus.NEEDS_AUTH):
             with ux_utils.print_exception_no_traceback():
                 raise exceptions.ApiServerConnectionError(endpoint) from exc
         # Lock to prevent multiple processes from starting the server at the

sky/server/constants.py

@@ -26,6 +26,8 @@ CLUSTER_REFRESH_DAEMON_INTERVAL_SECONDS = 60
 
 # Environment variable for a file path to the API cookie file.
 API_COOKIE_FILE_ENV_VAR = f'{constants.SKYPILOT_ENV_VAR_PREFIX}API_COOKIE_FILE'
+# Default file if unset.
+API_COOKIE_FILE_DEFAULT_LOCATION = '~/.sky/cookies.txt'
 
 # The path to the dashboard build output
 DASHBOARD_DIR = os.path.join(os.path.dirname(__file__), '..', 'dashboard',