skypilot-nightly 1.0.0.dev20250328__py3-none-any.whl → 1.0.0.dev20250330__py3-none-any.whl

sky/serve/server/core.py

@@ -25,7 +25,6 @@ from sky.utils import command_runner
 from sky.utils import common
 from sky.utils import common_utils
 from sky.utils import controller_utils
-from sky.utils import resources_utils
 from sky.utils import rich_utils
 from sky.utils import subprocess_utils
 from sky.utils import ux_utils
@@ -33,74 +32,6 @@ from sky.utils import ux_utils
 logger = sky_logging.init_logger(__name__)
 
 
-def _validate_service_task(task: 'sky.Task') -> None:
-    """Validate the task for Sky Serve.
-
-    Args:
-        task: sky.Task to validate
-
-    Raises:
-        ValueError: if the arguments are invalid.
-        RuntimeError: if the task.serve is not found.
-    """
-    spot_resources: List['sky.Resources'] = [
-        resource for resource in task.resources if resource.use_spot
-    ]
-    # TODO(MaoZiming): Allow mixed on-demand and spot specification in resources
-    # On-demand fallback should go to the resources specified as on-demand.
-    if len(spot_resources) not in [0, len(task.resources)]:
-        with ux_utils.print_exception_no_traceback():
-            raise ValueError(
-                'Resources must either all use spot or none use spot. '
-                'To use on-demand and spot instances together, '
-                'use `dynamic_ondemand_fallback` or set '
-                'base_ondemand_fallback_replicas.')
-
-    if task.service is None:
-        with ux_utils.print_exception_no_traceback():
-            raise RuntimeError('Service section not found.')
-
-    policy_description = ('on-demand'
-                          if task.service.dynamic_ondemand_fallback else 'spot')
-    for resource in list(task.resources):
-        if resource.job_recovery is not None:
-            with ux_utils.print_exception_no_traceback():
-                raise ValueError('job_recovery is disabled for SkyServe. '
-                                 'SkyServe will replenish preempted spot '
-                                 f'with {policy_description} instances.')
-
-    replica_ingress_port: Optional[int] = int(
-        task.service.ports) if (task.service.ports is not None) else None
-    for requested_resources in task.resources:
-        if (task.service.use_ondemand_fallback and
-                not requested_resources.use_spot):
-            with ux_utils.print_exception_no_traceback():
-                raise ValueError(
-                    '`use_ondemand_fallback` is only supported '
-                    'for spot resources. Please explicitly specify '
-                    '`use_spot: true` in resources for on-demand fallback.')
-        if task.service.ports is None:
-            requested_ports = list(
-                resources_utils.port_ranges_to_set(requested_resources.ports))
-            if len(requested_ports) != 1:
-                with ux_utils.print_exception_no_traceback():
-                    raise ValueError(
-                        'To open multiple ports on the replica, please set the '
-                        '`service.ports` field to specify a main service port. '
-                        'Must only specify one port in resources otherwise. '
-                        'Each replica will use the port specified as '
-                        'application ingress port.')
-            service_port = requested_ports[0]
-            if replica_ingress_port is None:
-                replica_ingress_port = service_port
-            elif service_port != replica_ingress_port:
-                with ux_utils.print_exception_no_traceback():
-                    raise ValueError(
-                        f'Got multiple ports: {service_port} and '
-                        f'{replica_ingress_port} in different resources. '
-                        'Please specify the same port instead.')
-
-
 def _rewrite_tls_credential_paths_and_get_tls_env_vars(
         service_name: str, task: 'sky.Task') -> Dict[str, Any]:
     """Rewrite the paths of TLS credentials in the task.
@@ -113,7 +44,7 @@ def _rewrite_tls_credential_paths_and_get_tls_env_vars(
         The generated template variables for TLS.
     """
     service_spec = task.service
-    # Already checked by _validate_service_task
+    # Already checked by validate_service_task
     assert service_spec is not None
     if service_spec.tls_credential is None:
         return {'use_tls': False}
@@ -166,7 +97,7 @@ def up(
                              'only contains lower letters, numbers and dash): '
                              f'{constants.CLUSTER_NAME_VALID_REGEX}')
 
-    _validate_service_task(task)
+    serve_utils.validate_service_task(task)
     # Always apply the policy again here, even though it might have been applied
     # in the CLI. This is to ensure that we apply the policy to the final DAG
     # and get the mutated config.
@@ -322,7 +253,7 @@ def up(
                 skip_status_check=True).get(lb_port)
             assert socket_endpoint is not None, (
                 'Did not get endpoint for controller.')
-            # Already checked by _validate_service_task
+            # Already checked by validate_service_task
             assert task.service is not None
             protocol = ('http'
                         if task.service.tls_credential is None else 'https')
@@ -376,7 +307,7 @@ def update(
         mode: Update mode.
     """
     task.validate()
-    _validate_service_task(task)
+    serve_utils.validate_service_task(task)
 
     # Always apply the policy again here, even though it might have been applied
     # in the CLI. This is to ensure that we apply the policy to the final DAG

sky/serve/service_spec.py

@@ -10,6 +10,7 @@ from sky.adaptors import common as adaptors_common
 from sky.serve import constants
 from sky.serve import load_balancing_policies as lb_policies
 from sky.serve import serve_utils
+from sky.serve import spot_placer as spot_placer_lib
 from sky.utils import common_utils
 from sky.utils import schemas
 from sky.utils import ux_utils
@@ -30,6 +31,7 @@ class SkyServiceSpec:
         readiness_timeout_seconds: int,
         min_replicas: int,
         max_replicas: Optional[int] = None,
+        num_overprovision: Optional[int] = None,
         ports: Optional[str] = None,
         target_qps_per_replica: Optional[float] = None,
         post_data: Optional[Dict[str, Any]] = None,
@@ -37,6 +39,7 @@ class SkyServiceSpec:
         readiness_headers: Optional[Dict[str, str]] = None,
         dynamic_ondemand_fallback: Optional[bool] = None,
         base_ondemand_fallback_replicas: Optional[int] = None,
+        spot_placer: Optional[str] = None,
         upscale_delay_seconds: Optional[int] = None,
         downscale_delay_seconds: Optional[int] = None,
         load_balancing_policy: Optional[str] = None,
@@ -78,6 +81,7 @@ class SkyServiceSpec:
         self._readiness_timeout_seconds: int = readiness_timeout_seconds
         self._min_replicas: int = min_replicas
         self._max_replicas: Optional[int] = max_replicas
+        self._num_overprovision: Optional[int] = num_overprovision
         self._ports: Optional[str] = ports
         self._target_qps_per_replica: Optional[float] = target_qps_per_replica
         self._post_data: Optional[Dict[str, Any]] = post_data
@@ -88,6 +92,7 @@ class SkyServiceSpec:
             bool] = dynamic_ondemand_fallback
         self._base_ondemand_fallback_replicas: Optional[
             int] = base_ondemand_fallback_replicas
+        self._spot_placer: Optional[str] = spot_placer
         self._upscale_delay_seconds: Optional[int] = upscale_delay_seconds
         self._downscale_delay_seconds: Optional[int] = downscale_delay_seconds
         self._load_balancing_policy: Optional[str] = load_balancing_policy
@@ -161,6 +166,7 @@ class SkyServiceSpec:
                 min_replicas = constants.DEFAULT_MIN_REPLICAS
             service_config['min_replicas'] = min_replicas
             service_config['max_replicas'] = None
+            service_config['num_overprovision'] = None
             service_config['target_qps_per_replica'] = None
             service_config['upscale_delay_seconds'] = None
             service_config['downscale_delay_seconds'] = None
@@ -168,6 +174,8 @@ class SkyServiceSpec:
             service_config['min_replicas'] = policy_section['min_replicas']
             service_config['max_replicas'] = policy_section.get(
                 'max_replicas', None)
+            service_config['num_overprovision'] = policy_section.get(
+                'num_overprovision', None)
             service_config['target_qps_per_replica'] = policy_section.get(
                 'target_qps_per_replica', None)
             service_config['upscale_delay_seconds'] = policy_section.get(
@@ -179,6 +187,8 @@ class SkyServiceSpec:
                     'base_ondemand_fallback_replicas', None)
             service_config['dynamic_ondemand_fallback'] = policy_section.get(
                 'dynamic_ondemand_fallback', None)
+            service_config['spot_placer'] = policy_section.get(
+                'spot_placer', None)
 
         service_config['load_balancing_policy'] = config.get(
             'load_balancing_policy', None)
@@ -238,12 +248,15 @@ class SkyServiceSpec:
         add_if_not_none('readiness_probe', 'headers', self._readiness_headers)
         add_if_not_none('replica_policy', 'min_replicas', self.min_replicas)
         add_if_not_none('replica_policy', 'max_replicas', self.max_replicas)
+        add_if_not_none('replica_policy', 'num_overprovision',
+                        self.num_overprovision)
         add_if_not_none('replica_policy', 'target_qps_per_replica',
                         self.target_qps_per_replica)
         add_if_not_none('replica_policy', 'dynamic_ondemand_fallback',
                         self.dynamic_ondemand_fallback)
         add_if_not_none('replica_policy', 'base_ondemand_fallback_replicas',
                         self.base_ondemand_fallback_replicas)
+        add_if_not_none('replica_policy', 'spot_placer', self.spot_placer)
         add_if_not_none('replica_policy', 'upscale_delay_seconds',
                         self.upscale_delay_seconds)
         add_if_not_none('replica_policy', 'downscale_delay_seconds',
@@ -269,6 +282,9 @@ class SkyServiceSpec:
         policy_strs: List[str] = []
         if (self.dynamic_ondemand_fallback is not None and
                 self.dynamic_ondemand_fallback):
+            if self.spot_placer is not None:
+                if self.spot_placer == spot_placer_lib.SPOT_HEDGE_PLACER:
+                    return 'SpotHedge'
             policy_strs.append('Dynamic on-demand fallback')
             if self.base_ondemand_fallback_replicas is not None:
                 policy_strs.append(
@@ -281,8 +297,12 @@ class SkyServiceSpec:
                 policy_strs.append('Static spot mixture with '
                                    f'{self.base_ondemand_fallback_replicas} '
                                    f'base on-demand replica{plural}')
+        if self.spot_placer is not None:
+            if not policy_strs:
+                policy_strs.append('Spot placement')
+            policy_strs.append(f'with {self.spot_placer} placer')
         if not policy_strs:
-            return 'No spot fallback policy'
+            return 'No spot policy'
         return ' '.join(policy_strs)
 
     def autoscaling_policy_str(self):
@@ -294,9 +314,13 @@ class SkyServiceSpec:
         assert self.target_qps_per_replica is not None
         # TODO(tian): Refactor to contain more information
         max_plural = '' if self.max_replicas == 1 else 's'
+        overprovision_str = ''
+        if self.num_overprovision is not None:
+            overprovision_str = (
+                f' with {self.num_overprovision} overprovisioned replicas')
         return (f'Autoscaling from {self.min_replicas} to {self.max_replicas} '
-                f'replica{max_plural} (target QPS per replica: '
-                f'{self.target_qps_per_replica})')
+                f'replica{max_plural}{overprovision_str} (target QPS per '
+                f'replica: {self.target_qps_per_replica})')
 
     def set_ports(self, ports: str) -> None:
         self._ports = ports
@@ -339,6 +363,10 @@ class SkyServiceSpec:
         # If None, treated as having the same value of min_replicas.
         return self._max_replicas
 
+    @property
+    def num_overprovision(self) -> Optional[int]:
+        return self._num_overprovision
+
     @property
     def ports(self) -> Optional[str]:
         return self._ports
@@ -372,6 +400,10 @@ class SkyServiceSpec:
     def dynamic_ondemand_fallback(self) -> Optional[bool]:
         return self._dynamic_ondemand_fallback
 
+    @property
+    def spot_placer(self) -> Optional[str]:
+        return self._spot_placer
+
     @property
     def upscale_delay_seconds(self) -> Optional[int]:
         return self._upscale_delay_seconds

sky/serve/spot_placer.py

@@ -0,0 +1,278 @@
+"""Spot Placer for SpotHedge."""
+
+import collections
+import dataclasses
+import enum
+import typing
+from typing import Any, Dict, List, Optional, Set
+
+from sky import check as sky_check
+from sky import clouds as sky_clouds
+from sky import sky_logging
+from sky.clouds import cloud as sky_cloud
+from sky.utils import registry
+from sky.utils import resources_utils
+from sky.utils import ux_utils
+
+if typing.TYPE_CHECKING:
+    from sky import resources as resources_lib
+    from sky import task as task_lib
+    from sky.serve import service_spec
+
+logger = sky_logging.init_logger(__name__)
+
+SPOT_PLACERS = {}
+DEFAULT_SPOT_PLACER = None
+SPOT_HEDGE_PLACER = 'dynamic_fallback'
+
+
+@dataclasses.dataclass
+class Location:
+    """Location class of a spot instance."""
+    cloud: 'sky_clouds.Cloud'
+    region: str
+    zone: Optional[str]
+
+    def __eq__(self, other) -> bool:
+        if isinstance(other, Location):
+            return (self.cloud.is_same_cloud(other.cloud) and
+                    self.region == other.region and self.zone == other.zone)
+        return False
+
+    def __hash__(self) -> int:
+        return hash(
+            str(self.cloud) + self.region +
+            (self.zone if self.zone is not None else ''))
+
+    @classmethod
+    def from_resources(cls, resources: 'resources_lib.Resources') -> 'Location':
+        return cls(resources.cloud, resources.region, resources.zone)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {'cloud': self.cloud, 'region': self.region, 'zone': self.zone}
+
+    @classmethod
+    def from_pickleable(
+        cls,
+        data: Optional[Dict[str, Optional[str]]],
+    ) -> Optional['Location']:
+        if data is None:
+            return None
+        cloud = registry.CLOUD_REGISTRY.from_str(data['cloud'])
+        assert cloud is not None
+        assert data['region'] is not None
+        return cls(
+            cloud=cloud,
+            region=data['region'],
+            zone=data['zone'],
+        )
+
+    def to_pickleable(self) -> Dict[str, Optional[str]]:
+        return {
+            'cloud': str(self.cloud),
+            'region': self.region,
+            'zone': self.zone,
+        }
+
+
+class LocationStatus(enum.Enum):
+    """Location Spot Status."""
+    ACTIVE = 'ACTIVE'
+    PREEMPTED = 'PREEMPTED'
+
+
+def _get_possible_location_from_task(task: 'task_lib.Task') -> List[Location]:
+
+    def _without_location(
+            resources: 'resources_lib.Resources') -> 'resources_lib.Resources':
+        return resources.copy(cloud=None, region=None, zone=None)
+
+    assert task.resources  # Guaranteed in task constructor
+    empty_location_resources = _without_location(list(task.resources)[0])
+    empty_location_resources_config = empty_location_resources.to_yaml_config()
+
+    location_requirements: Dict[str, Dict[str, Set[str]]] = (
+        collections.defaultdict(lambda: collections.defaultdict(set)))
+
+    for r in task.resources:
+        if (_without_location(r).to_yaml_config() !=
+                empty_location_resources_config):
+            with ux_utils.print_exception_no_traceback():
+                raise ValueError(
+                    'Different resource configurations are not supported '
+                    'for spot placement. All resources must have the same '
+                    'configuration except for cloud/region/zone.')
+        if r.cloud is None:
+            continue
+        cloud_str = str(r.cloud)
+        if r.region is None:
+            # Access defaultdict to create empty entry if it doesn't exist.
+            _ = location_requirements[cloud_str]
+            continue
+        if r.zone is None:
+            # Same as above.
+            _ = location_requirements[cloud_str][r.region]
+            continue
+        location_requirements[cloud_str][r.region].add(r.zone)
+
+    clouds_list: List[sky_clouds.Cloud] = []
+    for c in location_requirements.keys():
+        cloud_obj = registry.CLOUD_REGISTRY.from_str(c)
+        assert cloud_obj is not None
+        clouds_list.append(cloud_obj)
+    if not clouds_list:
+        # If the cloud list is empty, that means the user has no location
+        # related requirements. Then we start with all enabled clouds and
+        # all possible regions and zones.
+        clouds_list = sky_check.get_cached_enabled_clouds_or_refresh(
+            capability=sky_cloud.CloudCapability.COMPUTE,
+            raise_if_no_cloud_access=False)
+        for cloud in clouds_list:
+            # Create empty entry for each cloud.
+            _ = location_requirements[str(cloud)]
+
+    possible_locations = set()
+    for cloud in clouds_list:
+        feasible_resources: resources_utils.FeasibleResources = (
+            cloud.get_feasible_launchable_resources(empty_location_resources,
+                                                    num_nodes=task.num_nodes))
+        for feasible in feasible_resources.resources_list:
+            # We set override_optimize_by_zone=True to force the provisioner
+            # to use zone-level provisioning. This is to get accurate location
+            # information.
+            launchables: List['resources_lib.Resources'] = (
+                resources_utils.make_launchables_for_valid_region_zones(
+                    feasible, override_optimize_by_zone=True))
+            for launchable in launchables:
+                cloud_str = str(launchable.cloud)
+                region = launchable.region
+                zone = launchable.zone
+                if (cloud_str not in location_requirements and
+                        location_requirements):
+                    continue
+                # We need to use .get() here to avoid creating extra entries in
+                # location_requirements, and being treated as user's requirement
+                # in the following regions.
+                cloud_reqs = location_requirements.get(cloud_str, {})
+                if region not in cloud_reqs and cloud_reqs:
+                    continue
+                region_reqs = cloud_reqs.get(region, set())
+                if zone not in region_reqs and region_reqs:
+                    continue
+                possible_locations.add(Location.from_resources(launchable))
+
+    return list(possible_locations)
+
+
+class SpotPlacer:
+    """Spot Placement specification."""
+
+    def __init__(self, task: 'task_lib.Task') -> None:
+        possible_locations = _get_possible_location_from_task(task)
+        logger.info(f'{len(possible_locations)} possible location candidates '
+                    'are enabled for spot placement.')
+        logger.debug(f'All possible locations: {possible_locations}')
+        self.location2status: Dict[Location, LocationStatus] = {
+            location: LocationStatus.ACTIVE for location in possible_locations
+        }
+        self.location2cost: Dict[Location, float] = {}
+        # Already checked there is only one resource in the task.
+        self.resources = list(task.resources)[0]
+        self.num_nodes = task.num_nodes
+
+    def __init_subclass__(cls, name: str, default: bool = False):
+        SPOT_PLACERS[name] = cls
+        if default:
+            global DEFAULT_SPOT_PLACER
+            assert DEFAULT_SPOT_PLACER is None, (
+                'Only one policy can be default.')
+            DEFAULT_SPOT_PLACER = name
+
+    def select_next_location(self,
+                             current_locations: List[Location]) -> Location:
+        """Select next location to place spot instance."""
+        raise NotImplementedError
+
+    def set_active(self, location: Location) -> None:
+        assert location in self.location2status, location
+        self.location2status[location] = LocationStatus.ACTIVE
+
+    def set_preemptive(self, location: Location) -> None:
+        assert location in self.location2status, location
+        self.location2status[location] = LocationStatus.PREEMPTED
+
+    def clear_preemptive_locations(self) -> None:
+        for location in self.location2status:
+            self.location2status[location] = LocationStatus.ACTIVE
+
+    def _min_cost_location(self, locations: List[Location]) -> Location:
+
+        def _get_cost_per_hour(location: Location) -> float:
+            if location in self.location2cost:
+                return self.location2cost[location]
+            # TODO(tian): Is there a better way to do this? This is for filling
+            # instance type so the get_cost() can operate normally.
+            r: 'resources_lib.Resources' = self.resources.copy(
+                **location.to_dict())
+            assert r.cloud is not None
+            rs = r.cloud.get_feasible_launchable_resources(
+                r, num_nodes=self.num_nodes).resources_list
+            # For some clouds, there might have multiple instance types
+            # satisfying the resource request. In such case we choose the
+            # cheapest one, as the optimizer does. Reference:
+            # sky/optimizer.py::Optimizer::_print_candidates
+            cost = min(r.get_cost(seconds=3600) for r in rs)
+            self.location2cost[location] = cost
+            return cost
+
+        return min(locations, key=_get_cost_per_hour)
+
+    def _location_with_status(self, status: LocationStatus) -> List[Location]:
+        return [
+            location
+            for location, location_type in self.location2status.items()
+            if location_type == status
+        ]
+
+    def active_locations(self) -> List[Location]:
+        return self._location_with_status(LocationStatus.ACTIVE)
+
+    def preemptive_locations(self) -> List[Location]:
+        return self._location_with_status(LocationStatus.PREEMPTED)
+
+    @classmethod
+    def from_task(cls, spec: 'service_spec.SkyServiceSpec',
+                  task: 'task_lib.Task') -> Optional['SpotPlacer']:
+        if spec.spot_placer is None:
+            return None
+        return SPOT_PLACERS[spec.spot_placer](task)
+
+
+class DynamicFallbackSpotPlacer(SpotPlacer,
+                                name=SPOT_HEDGE_PLACER,
+                                default=True):
+    """Dynamic Fallback Placer."""
+
+    def select_next_location(self,
+                             current_locations: List[Location]) -> Location:
+        active_locations = self.active_locations()
+        # Prioritize locations that are not currently used.
+        candidate_locations: List[Location] = [
+            location for location in active_locations
+            if location not in current_locations
+        ]
+        # If no candidate locations, use all active locations.
+        if not candidate_locations:
+            candidate_locations = active_locations
+        res = self._min_cost_location(candidate_locations)
+        logger.info(f'Active locations: {active_locations}\n'
+                    f'Current locations: {current_locations}\n'
+                    f'Candidate locations: {candidate_locations}\n'
+                    f'Selected location: {res}\n')
+        return res
+
+    def set_preemptive(self, location: Location) -> None:
+        super().set_preemptive(location)
+        # Prevent the case with only one active location.
+        if len(self.active_locations()) < 2:
+            self.clear_preemptive_locations()

sky/server/common.py

@@ -51,6 +51,11 @@ API_SERVER_CMD = '-m sky.server.server'
 API_SERVER_CLIENT_DIR = pathlib.Path('~/.sky/api_server/clients')
 RETRY_COUNT_ON_TIMEOUT = 3
 
+# The maximum time to wait for the API server to start, set to a conservative
+# value that unlikely to reach since the server might be just starting slowly
+# (e.g. in high contention env) and we will exit eagerly if server exit.
+WAIT_APISERVER_START_TIMEOUT_SEC = 60
+
 SKY_API_VERSION_WARNING = (
     f'{colorama.Fore.YELLOW}SkyPilot API server is too old: '
     f'v{{server_version}} (client version is v{{client_version}}). '
@@ -179,7 +184,8 @@ def _start_api_server(deploy: bool = False,
     server_url = get_server_url(host)
     assert server_url in AVAILABLE_LOCAL_API_SERVER_URLS, (
         f'server url {server_url} is not a local url')
-    with rich_utils.client_status('Starting SkyPilot API server'):
+    with rich_utils.client_status('Starting SkyPilot API server, '
+                                  f'view logs at {constants.API_SERVER_LOGS}'):
         logger.info(f'{colorama.Style.DIM}Failed to connect to '
                     f'SkyPilot API server at {server_url}. '
                     'Starting a local server.'
@@ -216,14 +222,16 @@ def _start_api_server(deploy: bool = False,
         # If this is called from a CLI invocation, we need
         # start_new_session=True so that SIGINT on the CLI will not also kill
         # the API server.
-        subprocess.Popen(cmd, shell=True, start_new_session=True)
+        proc = subprocess.Popen(cmd, shell=True, start_new_session=True)
 
-        # Wait for the server to start until timeout.
-        # Conservative upper time bound for starting the server based on
-        # profiling.
-        timeout_sec = 12
         start_time = time.time()
         while True:
+            # Check if process has exited
+            if proc.poll() is not None:
+                with ux_utils.print_exception_no_traceback():
+                    raise RuntimeError(
+                        'SkyPilot API server process exited unexpectedly.\n'
+                        f'View logs at: {constants.API_SERVER_LOGS}')
             api_server_info = get_api_server_status()
             assert api_server_info.status != ApiServerStatus.VERSION_MISMATCH, (
                 f'API server version mismatch when starting the server. '
@@ -231,7 +239,7 @@ def _start_api_server(deploy: bool = False,
                 f'Client version: {server_constants.API_VERSION}')
             if api_server_info.status == ApiServerStatus.HEALTHY:
                 break
-            elif time.time() - start_time >= timeout_sec:
+            elif time.time() - start_time >= WAIT_APISERVER_START_TIMEOUT_SEC:
                 with ux_utils.print_exception_no_traceback():
                     raise RuntimeError(
                         'Failed to start SkyPilot API server at '

sky/server/requests/executor.py

@@ -465,7 +465,7 @@ def start(deploy: bool) -> List[multiprocessing.Process]:
             target=mp_queue.start_queue_manager, args=(queue_names, port))
         queue_server.start()
         sub_procs.append(queue_server)
-        mp_queue.wait_for_queues_to_be_ready(queue_names, port=port)
+        mp_queue.wait_for_queues_to_be_ready(queue_names, queue_server, port)
 
     logger.info('Request queues created')
 

sky/server/requests/queues/mp_queue.py

@@ -1,4 +1,5 @@
 """Shared queues for multiprocessing."""
+import multiprocessing
 from multiprocessing import managers
 import queue
 import time
@@ -57,10 +58,13 @@ def get_queue(queue_name: str,
 
 
 def wait_for_queues_to_be_ready(queue_names: List[str],
+                                queue_server: multiprocessing.Process,
                                 port: int = DEFAULT_QUEUE_MANAGER_PORT) -> None:
     """Wait for the queues to be ready after queue manager is just started."""
     initial_time = time.time()
-    max_wait_time = 5
+    # Wait for queue manager to be ready. Exit eagerly if the manager process
+    # exits, just wait for a long timeout that is unlikely to reach otherwise.
+    max_wait_time = 60
     while queue_names:
         try:
             get_queue(queue_names[0], port)
@@ -70,6 +74,9 @@ def wait_for_queues_to_be_ready(queue_names: List[str],
             logger.info(f'Waiting for request queue, named {queue_names[0]!r}, '
                         f'to be ready...')
             time.sleep(0.2)
+            if not queue_server.is_alive():
+                raise RuntimeError(
+                    'Queue manager process exited unexpectedly.') from e
             if time.time() - initial_time > max_wait_time:
                 raise RuntimeError(
                     f'Request queue, named {queue_names[0]!r}, '

sky/server/server.py

@@ -1116,7 +1116,7 @@ if __name__ == '__main__':
 
     sub_procs = []
     try:
-        sub_procs = executor.start(cmd_args.deploy)
+        sub_procs = executor.start(deploy=cmd_args.deploy)
         logger.info(f'Starting SkyPilot API server, workers={num_workers}')
         # We don't support reload for now, since it may cause leakage of request
         # workers or interrupt running requests.

sky/utils/registry.py

@@ -125,3 +125,5 @@ JOBS_RECOVERY_STRATEGY_REGISTRY: _Registry = (
         registry_name='jobs recovery strategy',
         exclude=None,
         type_register=True))
+
+# TODO(tian): Add a registry for spot placer.