skypilot-nightly 1.0.0.dev20250428__py3-none-any.whl → 1.0.0.dev20250429__py3-none-any.whl

sky/provision/docker_utils.py

@@ -28,6 +28,9 @@ SETUP_ENV_VARS_CMD = (
 # the command.
 DOCKER_PERMISSION_DENIED_STR = ('permission denied while trying to connect to '
                                 'the Docker daemon socket')
+
+DOCKER_SOCKET_NOT_READY_STR = ('Is the docker daemon running?')
+
 _DOCKER_SOCKET_WAIT_TIMEOUT_SECONDS = 30
 
 
@@ -173,22 +176,25 @@ class DockerInitializer:
                 stream_logs=False,
                 separate_stderr=separate_stderr,
                 log_path=self.log_path)
-            if (DOCKER_PERMISSION_DENIED_STR in stdout + stderr and
-                    wait_for_docker_daemon):
-                if time.time() - start > _DOCKER_SOCKET_WAIT_TIMEOUT_SECONDS:
-                    if rc == 0:
-                        # Set returncode to 1 if failed to connect to docker
-                        # daemon after timeout.
-                        rc = 1
-                    break
-                # Close the cached connection to make the permission update of
-                # ssh user take effect, e.g. usermod -aG docker $USER, called
-                # by cloud-init of Azure.
-                self.runner.close_cached_connection()
-                logger.info('Failed to connect to docker daemon. It might be '
-                            'initializing, retrying in 5 seconds...')
-                time.sleep(5)
-                continue
+            if (DOCKER_PERMISSION_DENIED_STR in stdout + stderr or
+                    DOCKER_SOCKET_NOT_READY_STR in stdout + stderr):
+                if wait_for_docker_daemon:
+                    if time.time(
+                    ) - start > _DOCKER_SOCKET_WAIT_TIMEOUT_SECONDS:
+                        if rc == 0:
+                            # Set returncode to 1 if failed to connect to docker
+                            # daemon after timeout.
+                            rc = 1
+                        break
+                    # Close the cached connection to make the permission update
+                    # of ssh user take effect, e.g. usermod -aG docker $USER,
+                    # called by cloud-init of Azure.
+                    self.runner.close_cached_connection()
+                    logger.info(
+                        'Failed to connect to docker daemon. It might be '
+                        'initializing, retrying in 5 seconds...')
+                    time.sleep(5)
+                    continue
             break
         subprocess_utils.handle_returncode(
             rc,

sky/provision/kubernetes/utils.py

@@ -243,7 +243,7 @@ class GPULabelFormatter:
         raise NotImplementedError
 
     @classmethod
-    def get_label_value(cls, accelerator: str) -> str:
+    def get_label_values(cls, accelerator: str) -> List[str]:
         """Given a GPU type, returns the label value to be used"""
         raise NotImplementedError
 
@@ -311,10 +311,10 @@ class SkyPilotLabelFormatter(GPULabelFormatter):
         return [cls.LABEL_KEY]
 
     @classmethod
-    def get_label_value(cls, accelerator: str) -> str:
+    def get_label_values(cls, accelerator: str) -> List[str]:
         # For SkyPilot formatter, we use the accelerator str directly.
         # See sky.utils.kubernetes.gpu_labeler.
-        return accelerator.lower()
+        return [accelerator.lower()]
 
     @classmethod
     def match_label_key(cls, label_key: str) -> bool:
@@ -351,8 +351,8 @@ class CoreWeaveLabelFormatter(GPULabelFormatter):
         return [cls.LABEL_KEY]
 
     @classmethod
-    def get_label_value(cls, accelerator: str) -> str:
-        return accelerator.upper()
+    def get_label_values(cls, accelerator: str) -> List[str]:
+        return [accelerator.upper()]
 
     @classmethod
     def match_label_key(cls, label_key: str) -> bool:
@@ -438,8 +438,8 @@ class GKELabelFormatter(GPULabelFormatter):
         return count_to_topology
 
     @classmethod
-    def get_label_value(cls, accelerator: str) -> str:
-        return get_gke_accelerator_name(accelerator)
+    def get_label_values(cls, accelerator: str) -> List[str]:
+        return [get_gke_accelerator_name(accelerator)]
 
     @classmethod
     def get_accelerator_from_label_value(cls, value: str) -> str:
@@ -472,7 +472,7 @@ class GFDLabelFormatter(GPULabelFormatter):
     https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/overview.html
 
     This LabelFormatter can't be used in autoscaling clusters since accelerators
-    may map to multiple label, so we're not implementing `get_label_value`
+    may map to multiple label, so we're not implementing `get_label_values`
     """
 
     LABEL_KEY = 'nvidia.com/gpu.product'
@@ -486,10 +486,10 @@ class GFDLabelFormatter(GPULabelFormatter):
         return [cls.LABEL_KEY]
 
     @classmethod
-    def get_label_value(cls, accelerator: str) -> str:
-        """An accelerator can map to many Nvidia GFD labels
-        (e.g., A100-80GB-PCIE vs. A100-SXM4-80GB).
-        As a result, we do not support get_label_value for GFDLabelFormatter."""
+    def get_label_values(cls, accelerator: str) -> List[str]:
+        # An accelerator can map to many Nvidia GFD labels
+        # (e.g., A100-80GB-PCIE vs. A100-SXM4-80GB).
+        # TODO implement get_label_values for GFDLabelFormatter
         raise NotImplementedError
 
     @classmethod
@@ -1032,15 +1032,17 @@ def check_instance_fits(context: Optional[str],
         # met.
         assert acc_count is not None, (acc_type, acc_count)
         try:
-            gpu_label_key, gpu_label_val, _, _ = (
-                get_accelerator_label_key_value(context, acc_type, acc_count))
+            gpu_label_key, gpu_label_values, _, _ = (
+                get_accelerator_label_key_values(context, acc_type, acc_count))
+            if gpu_label_values is None:
+                gpu_label_values = []
         except exceptions.ResourcesUnavailableError as e:
             # If GPU not found, return empty list and error message.
             return False, str(e)
         # Get the set of nodes that have the GPU type
         gpu_nodes = [
             node for node in nodes if gpu_label_key in node.metadata.labels and
-            node.metadata.labels[gpu_label_key] == gpu_label_val
+            node.metadata.labels[gpu_label_key] in gpu_label_values
         ]
         if not gpu_nodes:
             return False, f'No GPU nodes found with {acc_type} on the cluster'
@@ -1082,12 +1084,12 @@ def check_instance_fits(context: Optional[str],
         return fits, reason
 
 
-def get_accelerator_label_key_value(
+def get_accelerator_label_key_values(
     context: Optional[str],
     acc_type: str,
     acc_count: int,
     check_mode=False
-) -> Tuple[Optional[str], Optional[str], Optional[str], Optional[str]]:
+) -> Tuple[Optional[str], Optional[List[str]], Optional[str], Optional[str]]:
     """Returns the label key and value for the given GPU/TPU type.
 
     Args:
@@ -1141,7 +1143,7 @@ def get_accelerator_label_key_value(
             tpu_topology_label_key = formatter.get_tpu_topology_label_key()
             tpu_topology_label_value = formatter.get_tpu_topology_label_value(
                 acc_type, acc_count)
-        return formatter.get_label_key(acc_type), formatter.get_label_value(
+        return formatter.get_label_key(acc_type), formatter.get_label_values(
             acc_type), tpu_topology_label_key, tpu_topology_label_value
 
     has_gpus, cluster_resources = detect_accelerator_resource(context)
@@ -1220,12 +1222,12 @@ def get_accelerator_label_key_value(
                                 # different topologies that maps to identical
                                 # number of TPU chips.
                                 if tpu_topology_chip_count == acc_count:
-                                    return (label, value, topology_label_key,
+                                    return (label, [value], topology_label_key,
                                             topology_value)
                                 else:
                                     continue
                         else:
-                            return label, value, None, None
+                            return label, [value], None, None
 
             # If no node is found with the requested acc_type, raise error
             with ux_utils.print_exception_no_traceback():
@@ -1387,10 +1389,10 @@ def check_credentials(context: Optional[str],
             # `get_unlabeled_accelerator_nodes`.
             # Therefore, if `get_unlabeled_accelerator_nodes` detects unlabelled
             # nodes, we skip this check.
-            get_accelerator_label_key_value(context,
-                                            acc_type='',
-                                            acc_count=0,
-                                            check_mode=True)
+            get_accelerator_label_key_values(context,
+                                             acc_type='',
+                                             acc_count=0,
+                                             check_mode=True)
         except exceptions.ResourcesUnavailableError as e:
             # If GPUs are not available, we return cluster as enabled
             # (since it can be a CPU-only cluster) but we also return the

sky/server/common.py

@@ -333,7 +333,7 @@ def _start_api_server(deploy: bool = False,
                 break
 
         server_url = get_server_url(host)
-        dashboard_msg = (f'Dashboard: {get_dashboard_url(server_url)}')
+        dashboard_msg = ''
         api_server_info = get_api_server_status(server_url)
         if api_server_info.version == _DEV_VERSION:
             dashboard_msg += (
@@ -343,12 +343,15 @@ def _start_api_server(deploy: bool = False,
                 dashboard_msg += (
                     'Dashboard is not built, '
                     'to build: npm --prefix sky/dashboard install '
-                    '&& npm --prefix sky/dashboard run build')
+                    '&& npm --prefix sky/dashboard run build\n')
             else:
                 dashboard_msg += (
                     'Dashboard may be stale when installed from source, '
                     'to rebuild: npm --prefix sky/dashboard install '
-                    '&& npm --prefix sky/dashboard run build')
+                    '&& npm --prefix sky/dashboard run build\n')
+            dashboard_msg += (
+                f'{ux_utils.INDENT_LAST_SYMBOL}{colorama.Fore.GREEN}'
+                f'Dashboard: {get_dashboard_url(server_url)}')
             dashboard_msg += f'{colorama.Style.RESET_ALL}'
         logger.info(
             ux_utils.finishing_message(

sky/server/config.py

@@ -0,0 +1,184 @@
+"""SkyPilot API Server configuration."""
+
+import dataclasses
+import enum
+
+from sky import sky_logging
+from sky.server import constants as server_constants
+from sky.utils import common_utils
+
+# Constants based on profiling the peak memory usage while serving various
+# sky commands. These estimation are highly related to usage patterns
+# (clouds enabled, type of requests, etc. see `tests/load_tests` for details.),
+# the profiling covers major clouds and common usage patterns. For user has
+# deviated usage pattern, they can override the default estimation by
+# environment variables.
+# NOTE(dev): update these constants for each release according to the load
+# test results.
+# TODO(aylei): maintaining these constants is error-prone, we may need to
+# automatically tune parallelism at runtime according to system usage stats
+# in the future.
+_LONG_WORKER_MEM_GB = 0.4
+_SHORT_WORKER_MEM_GB = 0.25
+# To control the number of long workers.
+_CPU_MULTIPLIER_FOR_LONG_WORKERS = 2
+# Limit the number of long workers of local API server, since local server is
+# typically:
+# 1. launched automatically in an environment with high resource contention
+#    (e.g. Laptop)
+# 2. used by a single user
+_MAX_LONG_WORKERS_LOCAL = 4
+# Percentage of memory for long requests
+# from the memory reserved for SkyPilot.
+# This is to reserve some memory for short requests.
+_MAX_MEM_PERCENT_FOR_BLOCKING = 0.6
+# Minimal number of long workers to ensure responsiveness.
+_MIN_LONG_WORKERS = 1
+# Minimal number of short workers, there is a daemon task running on short
+# workers so at least 2 workers are needed to ensure responsiveness.
+_MIN_SHORT_WORKERS = 2
+
+# Default number of burstable workers for local API server. A heuristic number
+# that is large enough for most local cases.
+# TODO(aylei): the number of burstable workers should be auto-tuned based on the
+# system usage stats.
+_BURSTABLE_WORKERS_FOR_LOCAL = 1024
+
+logger = sky_logging.init_logger(__name__)
+
+
+class QueueBackend(enum.Enum):
+    # Local queue backend serves queues in each process locally, which has
+    # lower resource usage but the consumer must be in the same process, i.e.
+    # this only works in single-process mode.
+    LOCAL = 'local'
+    # Multi-process queue backend starts a dedicated process for serving queues.
+    MULTIPROCESSING = 'multiprocessing'
+    # TODO(zhwu): we can add redis backend in the future.
+
+
+@dataclasses.dataclass
+class WorkerConfig:
+    garanteed_parallelism: int
+    burstable_parallelism: int
+
+
+@dataclasses.dataclass
+class ServerConfig:
+    num_server_workers: int
+    long_worker_config: WorkerConfig
+    short_worker_config: WorkerConfig
+    queue_backend: QueueBackend
+
+
+def compute_server_config(deploy: bool) -> ServerConfig:
+    """Compute the server config based on environment.
+
+    We have different assumptions for the resources in different deployment
+    modes, which leads to different worker setups:
+
+    - Deployment mode (deploy=True), we assume the resources are dedicated to
+      the API server and the resources will be tuned for serious use cases, so:
+      - Use multiprocessing queue backend and dedicated workers processes to
+        avoid GIL contention.
+      - Parallelism (number of executor processes) is fixed and executor
+        processes have same lifecycle with the server, which ensures
+        best-effort cache reusing and stable resources consumption.
+      - Reject to start in low resource environments, to avoid flaky
+        deployments.
+    - Local mode (deploy=False), we assume the server is running in a shared
+      environment (e.g. laptop) and users typically do not pay attention to
+      the resource setup of the server. Moreover, existing users may expect
+      some consistent behaviors with old versions, i.e. before API server was
+      introduced, so:
+      - The max number of long-running executor processes are limited, to avoid
+        high memory consumption when the server is idle.
+      - Allow burstable workers to handle requests when all long-running
+        workers are busy, which mimics the behavior of local sky CLI before
+        API server was introduced.
+      - Works in low resources environments, and further reduce the memory
+        consumption in low resource environments.
+
+    Note that there is still significant overhead for SDK users when migrate to
+    local API server. Since the users are free to run sky operations in Threads
+    when using SDK but all client operations will occupy at least one worker
+    process after API server was introduced.
+    """
+    cpu_count = common_utils.get_cpu_count()
+    mem_size_gb = common_utils.get_mem_size_gb()
+    max_parallel_for_long = _max_long_worker_parallism(cpu_count,
+                                                       mem_size_gb,
+                                                       local=not deploy)
+    max_parallel_for_short = _max_short_worker_parallism(
+        mem_size_gb, max_parallel_for_long)
+    queue_backend = QueueBackend.MULTIPROCESSING
+    burstable_parallel_for_long = 0
+    burstable_parallel_for_short = 0
+    num_server_workers = cpu_count
+    if not deploy:
+        # For local mode, use local queue backend since we only run 1 uvicorn
+        # worker in local mode and no multiprocessing is needed.
+        num_server_workers = 1
+        queue_backend = QueueBackend.LOCAL
+        # Enable burstable workers for local API server.
+        burstable_parallel_for_long = _BURSTABLE_WORKERS_FOR_LOCAL
+        burstable_parallel_for_short = _BURSTABLE_WORKERS_FOR_LOCAL
+        # Runs in low resource mode if the available memory is less than
+        # server_constants.MIN_AVAIL_MEM_GB.
+        if not deploy and mem_size_gb < server_constants.MIN_AVAIL_MEM_GB:
+            # Permanent worker process may have significant memory consumption
+            # (~350MB per worker) after running commands like `sky check`, so we
+            # don't start any permanent workers in low resource local mode. This
+            # mimics the behavior of local sky CLI before API server was
+            # introduced, where the CLI will start new process everytime and
+            # never reject to start due to resource constraints.
+            # Note that the refresh daemon will still occupy one worker
+            # permanently because it never exits.
+            max_parallel_for_long = 0
+            max_parallel_for_short = 0
+            logger.warning(
+                'SkyPilot API server will run in low resource mode because '
+                'the available memory is less than '
+                f'{server_constants.MIN_AVAIL_MEM_GB}GB.')
+    logger.info(
+        f'SkyPilot API server will start {num_server_workers} server processes '
+        f'with {max_parallel_for_long} background workers for long requests '
+        f'and will allow at max {max_parallel_for_short} short requests in '
+        f'parallel.')
+    return ServerConfig(
+        num_server_workers=num_server_workers,
+        queue_backend=queue_backend,
+        long_worker_config=WorkerConfig(
+            garanteed_parallelism=max_parallel_for_long,
+            burstable_parallelism=burstable_parallel_for_long),
+        short_worker_config=WorkerConfig(
+            garanteed_parallelism=max_parallel_for_short,
+            burstable_parallelism=burstable_parallel_for_short),
+    )
+
+
+def _max_long_worker_parallism(cpu_count: int,
+                               mem_size_gb: float,
+                               local=False) -> int:
+    """Max parallelism for long workers."""
+    # Reserve min available memory to avoid OOM.
+    available_mem = max(0, mem_size_gb - server_constants.MIN_AVAIL_MEM_GB)
+    cpu_based_max_parallel = cpu_count * _CPU_MULTIPLIER_FOR_LONG_WORKERS
+    mem_based_max_parallel = int(available_mem * _MAX_MEM_PERCENT_FOR_BLOCKING /
+                                 _LONG_WORKER_MEM_GB)
+    n = max(_MIN_LONG_WORKERS,
+            min(cpu_based_max_parallel, mem_based_max_parallel))
+    if local:
+        return min(n, _MAX_LONG_WORKERS_LOCAL)
+    return n
+
+
+def _max_short_worker_parallism(mem_size_gb: float,
+                                long_worker_parallism: int) -> int:
+    """Max parallelism for short workers."""
+    # Reserve memory for long workers and min available memory.
+    reserved_mem = server_constants.MIN_AVAIL_MEM_GB + (long_worker_parallism *
+                                                        _LONG_WORKER_MEM_GB)
+    available_mem = max(0, mem_size_gb - reserved_mem)
+    n = max(_MIN_SHORT_WORKERS, int(available_mem / _SHORT_WORKER_MEM_GB))
+    return n

sky/server/requests/executor.py

@@ -19,7 +19,6 @@ The number of the workers is determined by the system resources.
 See the [README.md](../README.md) for detailed architecture of the executor.
 """
 import contextlib
-import enum
 import multiprocessing
 import os
 import queue as queue_lib
@@ -37,6 +36,7 @@ from sky import models
 from sky import sky_logging
 from sky import skypilot_config
 from sky.server import common as server_common
+from sky.server import config as server_config
 from sky.server import constants as server_constants
 from sky.server.requests import payloads
 from sky.server.requests import preconditions
@@ -70,53 +70,6 @@ logger = sky_logging.init_logger(__name__)
 # platforms, including macOS.
 multiprocessing.set_start_method('spawn', force=True)
 
-# Constants based on profiling the peak memory usage while serving various
-# sky commands. These estimation are highly related to usage patterns
-# (clouds enabled, type of requests, etc. see `tests/load_tests` for details.),
-# the profiling covers major clouds and common usage patterns. For user has
-# deviated usage pattern, they can override the default estimation by
-# environment variables.
-# NOTE(dev): update these constants for each release according to the load
-# test results.
-# TODO(aylei): maintaining these constants is error-prone, we may need to
-# automatically tune parallelism at runtime according to system usage stats
-# in the future.
-_LONG_WORKER_MEM_GB = 0.4
-_SHORT_WORKER_MEM_GB = 0.25
-# To control the number of long workers.
-_CPU_MULTIPLIER_FOR_LONG_WORKERS = 2
-# Limit the number of long workers of local API server, since local server is
-# typically:
-# 1. launched automatically in an environment with high resource contention
-#    (e.g. Laptop)
-# 2. used by a single user
-_MAX_LONG_WORKERS_LOCAL = 4
-# Percentage of memory for long requests
-# from the memory reserved for SkyPilot.
-# This is to reserve some memory for short requests.
-_MAX_MEM_PERCENT_FOR_BLOCKING = 0.6
-# Minimal number of long workers to ensure responsiveness.
-_MIN_LONG_WORKERS = 1
-# Minimal number of short workers, there is a daemon task running on short
-# workers so at least 2 workers are needed to ensure responsiveness.
-_MIN_SHORT_WORKERS = 2
-
-# Default number of burstable workers for local API server. A heuristic number
-# that is large enough for most local cases.
-# TODO(aylei): the number of burstable workers should be auto-tuned based on the
-# system usage stats.
-_BURSTABLE_WORKERS_FOR_LOCAL = 1024
-
-
-class QueueBackend(enum.Enum):
-    # Local queue backend serves queues in each process locally, which has
-    # lower resource usage but the consumer must be in the same process, i.e.
-    # this only works in single-process mode.
-    LOCAL = 'local'
-    # Multi-process queue backend starts a dedicated process for serving queues.
-    MULTIPROCESSING = 'multiprocessing'
-    # TODO(zhwu): we can add redis backend in the future.
-
 
 class RequestQueue:
     """The queue for the requests, either redis or multiprocessing.
@@ -126,12 +79,12 @@ class RequestQueue:
 
     def __init__(self,
                  schedule_type: api_requests.ScheduleType,
-                 backend: Optional[QueueBackend] = None) -> None:
+                 backend: Optional[server_config.QueueBackend] = None) -> None:
         self.name = schedule_type.value
         self.backend = backend
-        if backend == QueueBackend.MULTIPROCESSING:
+        if backend == server_config.QueueBackend.MULTIPROCESSING:
             self.queue = mp_queue.get_queue(self.name)
-        elif backend == QueueBackend.LOCAL:
+        elif backend == server_config.QueueBackend.LOCAL:
             self.queue = local_queue.get_queue(self.name)
         else:
             raise RuntimeError(f'Invalid queue backend: {backend}')
@@ -162,7 +115,7 @@ class RequestQueue:
         return self.queue.qsize()
 
 
-queue_backend = QueueBackend.MULTIPROCESSING
+queue_backend = server_config.QueueBackend.MULTIPROCESSING
 
 
 def executor_initializer(proc_group: str):
@@ -186,13 +139,11 @@ class RequestWorker:
     # if there are available CPU/memory resources.
     burstable_parallelism: int = 0
 
-    def __init__(self,
-                 schedule_type: api_requests.ScheduleType,
-                 garanteed_parallelism: int,
-                 burstable_parallelism: int = 0) -> None:
+    def __init__(self, schedule_type: api_requests.ScheduleType,
+                 config: server_config.WorkerConfig) -> None:
         self.schedule_type = schedule_type
-        self.garanteed_parallelism = garanteed_parallelism
-        self.burstable_parallelism = burstable_parallelism
+        self.garanteed_parallelism = config.garanteed_parallelism
+        self.burstable_parallelism = config.burstable_parallelism
 
     def __str__(self) -> str:
         return f'Worker(schedule_type={self.schedule_type.value})'
@@ -455,80 +406,17 @@ def schedule_request(
         enqueue()
 
 
-def start(deploy: bool) -> List[multiprocessing.Process]:
+def start(config: server_config.ServerConfig) -> List[multiprocessing.Process]:
     """Start the request workers.
 
     Request workers run in background, schedule the requests and delegate the
-    request execution to executor processes. We have different assumptions for
-    the resources in different deployment modes, which leads to different
-    worker setups:
-
-    - Deployment mode (deploy=True), we assume the resources are dedicated to
-      the API server and the resources will be tuned for serious use cases, so:
-      - Use multiprocessing queue backend and dedicated workers processes to
-        avoid GIL contention.
-      - Parallelism (number of executor processes) is fixed and executor
-        processes have same lifecycle with the server, which ensures
-        best-effort cache reusing and stable resources consumption.
-      - Reject to start in low resource environments, to avoid flaky
-        deployments.
-    - Local mode (deploy=False), we assume the server is running in a shared
-      environment (e.g. laptop) and users typically do not pay attention to
-      the resource setup of the server. Moreover, existing users may expect
-      some consistent behaviors with old versions, i.e. before API server was
-      introduced, so:
-      - The max number of long-running executor processes are limited, to avoid
-        high memory consumption when the server is idle.
-      - Allow burstable workers to handle requests when all long-running
-        workers are busy, which mimics the behavior of local sky CLI before
-        API server was introduced.
-      - Works in low resources environments, and further reduce the memory
-        consumption in low resource environments.
-
-    Note that there is still significant overhead for SDK users when migrate to
-    local API server. Since the users are free to run sky operations in Threads
-    when using SDK but all client operations will occupy at least one worker
-    process after API server was introduced.
+    request execution to executor processes.
     """
-    # Determine the job capacity of the workers based on the system resources.
-    cpu_count = common_utils.get_cpu_count()
-    mem_size_gb = common_utils.get_mem_size_gb()
-    mem_for_workers = max(0, mem_size_gb - server_constants.MIN_AVAIL_MEM_GB)
-    # Runs in low resource mode if the available memory is less than
-    # server_constants.MIN_AVAIL_MEM_GB.
-    max_parallel_for_long = _max_long_worker_parallism(cpu_count,
-                                                       mem_for_workers,
-                                                       local=not deploy)
-    max_parallel_for_short = _max_short_worker_parallism(
-        mem_for_workers, max_parallel_for_long)
-    if not deploy and mem_size_gb < server_constants.MIN_AVAIL_MEM_GB:
-        # Permanent worker process may have significant memory consumption
-        # (~350MB per worker) after running commands like `sky check`, so we
-        # don't start any permanent workers in low resource local mode. This
-        # mimics the behavior of local sky CLI before API server was
-        # introduced, where the CLI will start new process everytime and
-        # never reject to start due to resource constraints.
-        # Note that the refresh daemon will still occupy one worker
-        # permanently because it never exits.
-        max_parallel_for_long = 0
-        max_parallel_for_short = 0
-        logger.warning(
-            'SkyPilot API server will run in low resource mode because '
-            'the available memory is less than '
-            f'{server_constants.MIN_AVAIL_MEM_GB}GB.')
-    else:
-        logger.info(
-            f'SkyPilot API server will start {max_parallel_for_long} workers '
-            f'for long requests and will allow at max '
-            f'{max_parallel_for_short} short requests in parallel.')
-    if not deploy:
-        # For local mode, use local queue backend since we only run 1 uvicorn
-        # worker in local mode.
-        global queue_backend
-        queue_backend = QueueBackend.LOCAL
+    global queue_backend
+    queue_backend = config.queue_backend
     sub_procs = []
     # Setup the queues.
-    if queue_backend == QueueBackend.MULTIPROCESSING:
+    if queue_backend == server_config.QueueBackend.MULTIPROCESSING:
         logger.info('Creating shared request queues')
         queue_names = [
             schedule_type.value for schedule_type in api_requests.ScheduleType
@@ -547,7 +435,7 @@ def start(deploy: bool) -> List[multiprocessing.Process]:
         mp_queue.wait_for_queues_to_be_ready(queue_names,
                                              queue_server,
                                              port=port)
-    elif queue_backend == QueueBackend.LOCAL:
+    elif queue_backend == server_config.QueueBackend.LOCAL:
         # No setup is needed for local queue backend.
         pass
     else:
@@ -563,40 +451,13 @@ def start(deploy: bool) -> List[multiprocessing.Process]:
         thread = threading.Thread(target=worker.run, daemon=True)
         thread.start()
 
-    burstable_parallelism = _BURSTABLE_WORKERS_FOR_LOCAL if not deploy else 0
     # Start a worker for long requests.
     long_worker = RequestWorker(schedule_type=api_requests.ScheduleType.LONG,
-                                garanteed_parallelism=max_parallel_for_long,
-                                burstable_parallelism=burstable_parallelism)
+                                config=config.long_worker_config)
     run_worker_in_background(long_worker)
 
     # Start a worker for short requests.
     short_worker = RequestWorker(schedule_type=api_requests.ScheduleType.SHORT,
-                                 garanteed_parallelism=max_parallel_for_short,
-                                 burstable_parallelism=burstable_parallelism)
+                                 config=config.short_worker_config)
     run_worker_in_background(short_worker)
     return sub_procs
-
-
-@annotations.lru_cache(scope='global', maxsize=1)
-def _max_long_worker_parallism(cpu_count: int,
-                               mem_size_gb: float,
-                               local=False) -> int:
-    """Max parallelism for long workers."""
-    cpu_based_max_parallel = cpu_count * _CPU_MULTIPLIER_FOR_LONG_WORKERS
-    mem_based_max_parallel = int(mem_size_gb * _MAX_MEM_PERCENT_FOR_BLOCKING /
-                                 _LONG_WORKER_MEM_GB)
-    n = max(_MIN_LONG_WORKERS,
-            min(cpu_based_max_parallel, mem_based_max_parallel))
-    if local:
-        return min(n, _MAX_LONG_WORKERS_LOCAL)
-    return n
-
-
-@annotations.lru_cache(scope='global', maxsize=1)
-def _max_short_worker_parallism(mem_size_gb: float,
-                                long_worker_parallism: int) -> int:
-    """Max parallelism for short workers."""
-    available_mem = mem_size_gb - (long_worker_parallism * _LONG_WORKER_MEM_GB)
-    n = max(_MIN_SHORT_WORKERS, int(available_mem / _SHORT_WORKER_MEM_GB))
-    return n