skypilot-nightly 1.0.0.dev20250623__py3-none-any.whl → 1.0.0.dev20250625__py3-none-any.whl

sky/server/common.py

@@ -9,11 +9,13 @@ import json
 import os
 import pathlib
 import re
+import shutil
 import subprocess
 import sys
+import tempfile
 import time
 import typing
-from typing import Any, Dict, Literal, Optional, Tuple
+from typing import Any, Dict, Literal, Optional, Tuple, Union
 from urllib import parse
 import uuid
 
@@ -27,6 +29,7 @@ from sky import skypilot_config
 from sky.adaptors import common as adaptors_common
 from sky.data import data_utils
 from sky.server import constants as server_constants
+from sky.server import rest
 from sky.skylet import constants
 from sky.usage import usage_lib
 from sky.utils import annotations
@@ -240,9 +243,9 @@ def get_api_server_status(endpoint: Optional[str] = None) -> ApiServerInfo:
     server_url = endpoint if endpoint is not None else get_server_url()
     while time_out_try_count <= RETRY_COUNT_ON_TIMEOUT:
         try:
-            response = requests.get(f'{server_url}/api/health',
-                                    timeout=2.5,
-                                    cookies=get_api_cookie_jar())
+            response = rest.get(f'{server_url}/api/health',
+                                timeout=2.5,
+                                cookies=get_api_cookie_jar())
         except requests.exceptions.Timeout:
             if time_out_try_count == RETRY_COUNT_ON_TIMEOUT:
                 return ApiServerInfo(status=ApiServerStatus.UNHEALTHY)
@@ -327,6 +330,8 @@ def get_request_id(response: 'requests.Response') -> RequestId:
 def _start_api_server(deploy: bool = False,
                       host: str = '127.0.0.1',
                       foreground: bool = False,
+                      metrics: bool = False,
+                      metrics_port: Optional[int] = None,
                       enable_basic_auth: bool = False):
     """Starts a SkyPilot API server locally."""
     server_url = get_server_url(host)
@@ -357,10 +362,13 @@ def _start_api_server(deploy: bool = False,
             args += ['--deploy']
         if host is not None:
             args += [f'--host={host}']
+        if metrics_port is not None:
+            args += [f'--metrics-port={metrics_port}']
 
         if foreground:
             # Replaces the current process with the API server
             os.environ[constants.ENV_VAR_IS_SKYPILOT_SERVER] = 'true'
+            _set_metrics_env_var(os.environ, metrics, deploy)
             if enable_basic_auth:
                 os.environ[constants.ENV_VAR_ENABLE_BASIC_AUTH] = 'true'
             os.execvp(args[0], args)
@@ -368,6 +376,10 @@ def _start_api_server(deploy: bool = False,
         log_path = os.path.expanduser(constants.API_SERVER_LOGS)
         os.makedirs(os.path.dirname(log_path), exist_ok=True)
 
+        # For spawn mode, copy the environ to avoid polluting the SDK process.
+        server_env = os.environ.copy()
+        server_env[constants.ENV_VAR_IS_SKYPILOT_SERVER] = 'true'
+        _set_metrics_env_var(server_env, metrics, deploy)
         # Start the API server process in the background and don't wait for it.
         # If this is called from a CLI invocation, we need
         # start_new_session=True so that SIGINT on the CLI will not also kill
@@ -437,6 +449,26 @@ def _start_api_server(deploy: bool = False,
                 f'SkyPilot API server started. {dashboard_msg}'))
 
 
+def _set_metrics_env_var(env: Union[Dict[str, str], os._Environ], metrics: bool,
+                         deploy: bool):
+    """Sets the metrics environment variables.
+
+    Args:
+        env: The environment variables to set.
+        metrics: Whether to enable metrics.
+        deploy: Whether the server is running in deploy mode, which means
+            multiple processes might be running.
+    """
+    if metrics:
+        env[constants.ENV_VAR_SERVER_METRICS_ENABLED] = 'true'
+        if deploy:
+            metrics_dir = os.path.join(tempfile.gettempdir(), 'metrics')
+            shutil.rmtree(metrics_dir, ignore_errors=True)
+            os.makedirs(metrics_dir, exist_ok=True)
+            # Refer to https://prometheus.github.io/client_python/multiprocess/
+            env['PROMETHEUS_MULTIPROC_DIR'] = metrics_dir
+
+
 def check_server_healthy(
     endpoint: Optional[str] = None
 ) -> Tuple[Literal[
@@ -571,6 +603,8 @@ def get_skypilot_version_on_disk() -> str:
 def check_server_healthy_or_start_fn(deploy: bool = False,
                                      host: str = '127.0.0.1',
                                      foreground: bool = False,
+                                     metrics: bool = False,
+                                     metrics_port: Optional[int] = None,
                                      enable_basic_auth: bool = False):
     api_server_status = None
     try:
@@ -592,7 +626,8 @@ def check_server_healthy_or_start_fn(deploy: bool = False,
             # have started the server while we were waiting for the lock.
             api_server_info = get_api_server_status(endpoint)
             if api_server_info.status == ApiServerStatus.UNHEALTHY:
-                _start_api_server(deploy, host, foreground, enable_basic_auth)
+                _start_api_server(deploy, host, foreground, metrics,
+                                  metrics_port, enable_basic_auth)
 
 
 def check_server_healthy_or_start(func):

sky/server/constants.py

@@ -7,7 +7,7 @@ from sky.skylet import constants
 # API server version, whenever there is a change in API server that requires a
 # restart of the local API server or error out when the client does not match
 # the server version.
-API_VERSION = '9'
+API_VERSION = '10'
 
 # Prefix for API request names.
 REQUEST_NAME_PREFIX = 'sky.'
@@ -22,6 +22,10 @@ API_SERVER_REQUEST_DB_PATH = '~/.sky/api_server/requests.db'
 # background.
 CLUSTER_REFRESH_DAEMON_INTERVAL_SECONDS = 60
 
+# The interval (seconds) for the volume status to be refreshed in the
+# background.
+VOLUME_REFRESH_DAEMON_INTERVAL_SECONDS = 60
+
 # Environment variable for a file path to the API cookie file.
 # Keep in sync with websocket_proxy.py
 API_COOKIE_FILE_ENV_VAR = f'{constants.SKYPILOT_ENV_VAR_PREFIX}API_COOKIE_FILE'

sky/server/metrics.py

@@ -0,0 +1,105 @@
+"""Instrumentation for the API server."""
+
+import asyncio
+import os
+import time
+
+import fastapi
+from prometheus_client import generate_latest
+from prometheus_client import multiprocess
+import prometheus_client as prom
+import starlette.middleware.base
+import uvicorn
+
+from sky import sky_logging
+
+logger = sky_logging.init_logger(__name__)
+
+# Total number of API server requests, grouped by path, method, and status.
+sky_apiserver_requests_total = prom.Counter(
+    'sky_apiserver_requests_total',
+    'Total number of API server requests',
+    ['path', 'method', 'status'],
+)
+
+# Time spent processing API server requests, grouped by path, method, and
+# status.
+sky_apiserver_request_duration_seconds = prom.Histogram(
+    'sky_apiserver_request_duration_seconds',
+    'Time spent processing API server requests',
+    ['path', 'method', 'status'],
+    buckets=(0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,
+             float('inf')),
+)
+
+metrics_app = fastapi.FastAPI()
+
+
+@metrics_app.get('/metrics')
+async def metrics() -> fastapi.Response:
+    """Expose aggregated Prometheus metrics from all worker processes."""
+    if os.environ.get('PROMETHEUS_MULTIPROC_DIR'):
+        # In multiprocess mode, we need to collect metrics from all processes.
+        registry = prom.CollectorRegistry()
+        multiprocess.MultiProcessCollector(registry)
+        data = generate_latest(registry)
+    else:
+        data = generate_latest()
+    return fastapi.Response(content=data,
+                            media_type=prom.CONTENT_TYPE_LATEST,
+                            headers={'Cache-Control': 'no-cache'})
+
+
+def run_metrics_server(host: str, port: int):
+    metrics_config = uvicorn.Config(
+        'sky.server.metrics:metrics_app',
+        host=host,
+        port=port,
+        workers=1,
+    )
+    metrics_server_instance = uvicorn.Server(metrics_config)
+    asyncio.run(metrics_server_instance.serve())
+
+
+def _get_status_code_group(status_code: int) -> str:
+    """Group status codes into classes (2xx, 5xx) to reduce cardinality."""
+    return f'{status_code // 100}xx'
+
+
+def _is_streaming_api(path: str) -> bool:
+    """Check if the path is a streaming API."""
+    path = path.rstrip('/')
+    return path.endswith('/logs') or path.endswith('/api/stream')
+
+
+class PrometheusMiddleware(starlette.middleware.base.BaseHTTPMiddleware):
+    """Middleware to collect Prometheus metrics for HTTP requests."""
+
+    async def dispatch(self, request: fastapi.Request, call_next):
+        path = request.url.path
+        logger.info(f'PROM Middleware Request: {request}, {request.url.path}')
+        streaming = _is_streaming_api(path)
+        if not streaming:
+            # Exclude streaming APIs, the duration is not meaningful.
+            # TODO(aylei): measure the duration of async execution instead.
+            start_time = time.time()
+        method = request.method
+        status_code_group = ''
+
+        try:
+            response = await call_next(request)
+            status_code_group = _get_status_code_group(response.status_code)
+        except Exception:  # pylint: disable=broad-except
+            status_code_group = '5xx'
+            raise
+        finally:
+            sky_apiserver_requests_total.labels(path=path,
+                                                method=method,
+                                                status=status_code_group).inc()
+            if not streaming:
+                duration = time.time() - start_time
+                sky_apiserver_request_duration_seconds.labels(
+                    path=path, method=method,
+                    status=status_code_group).observe(duration)
+
+        return response

sky/server/requests/executor.py

@@ -149,10 +149,25 @@ class RequestWorker:
         self.schedule_type = schedule_type
         self.garanteed_parallelism = config.garanteed_parallelism
         self.burstable_parallelism = config.burstable_parallelism
+        self._thread: Optional[threading.Thread] = None
+        self._cancel_event = threading.Event()
 
     def __str__(self) -> str:
         return f'Worker(schedule_type={self.schedule_type.value})'
 
+    def run_in_background(self) -> None:
+        # Thread dispatcher is sufficient for current scale, refer to
+        # tests/load_tests/test_queue_dispatcher.py for more details.
+        # Use daemon thread for automatic cleanup.
+        thread = threading.Thread(target=self.run, daemon=True)
+        thread.start()
+        self._thread = thread
+
+    def cancel(self) -> None:
+        if self._thread is not None:
+            self._cancel_event.set()
+            self._thread.join()
+
     def process_request(self, executor: process.BurstableExecutor,
                         queue: RequestQueue) -> None:
         try:
@@ -219,7 +234,7 @@ class RequestWorker:
                 burst_workers=self.burstable_parallelism,
                 initializer=executor_initializer,
                 initargs=(proc_group,))
-            while True:
+            while not self._cancel_event.is_set():
                 self.process_request(executor, queue)
         # TODO(aylei): better to distinct between KeyboardInterrupt and SIGTERM.
         except KeyboardInterrupt:
@@ -539,15 +554,21 @@ def schedule_request(request_id: str,
         enqueue()
 
 
-def start(config: server_config.ServerConfig) -> List[multiprocessing.Process]:
+def start(
+    config: server_config.ServerConfig
+) -> Tuple[Optional[multiprocessing.Process], List[RequestWorker]]:
     """Start the request workers.
 
     Request workers run in background, schedule the requests and delegate the
     request execution to executor processes.
+
+    Returns:
+        A tuple of the queue server process and the list of request worker
+        threads.
     """
     global queue_backend
     queue_backend = config.queue_backend
-    sub_procs = []
+    queue_server = None
     # Setup the queues.
     if queue_backend == server_config.QueueBackend.MULTIPROCESSING:
         logger.info('Creating shared request queues')
@@ -564,7 +585,6 @@ def start(config: server_config.ServerConfig) -> List[multiprocessing.Process]:
         queue_server = multiprocessing.Process(
             target=mp_queue.start_queue_manager, args=(queue_names, port))
         queue_server.start()
-        sub_procs.append(queue_server)
         mp_queue.wait_for_queues_to_be_ready(queue_names,
                                              queue_server,
                                              port=port)
@@ -577,20 +597,16 @@ def start(config: server_config.ServerConfig) -> List[multiprocessing.Process]:
 
     logger.info('Request queues created')
 
-    def run_worker_in_background(worker: RequestWorker):
-        # Thread dispatcher is sufficient for current scale, refer to
-        # tests/load_tests/test_queue_dispatcher.py for more details.
-        # Use daemon thread for automatic cleanup.
-        thread = threading.Thread(target=worker.run, daemon=True)
-        thread.start()
-
+    workers = []
     # Start a worker for long requests.
     long_worker = RequestWorker(schedule_type=api_requests.ScheduleType.LONG,
                                 config=config.long_worker_config)
-    run_worker_in_background(long_worker)
+    long_worker.run_in_background()
+    workers.append(long_worker)
 
     # Start a worker for short requests.
     short_worker = RequestWorker(schedule_type=api_requests.ScheduleType.SHORT,
                                  config=config.short_worker_config)
-    run_worker_in_background(short_worker)
-    return sub_procs
+    short_worker.run_in_background()
+    workers.append(short_worker)
+    return queue_server, workers

sky/server/requests/payloads.py

@@ -5,7 +5,6 @@ kwargs for the payloads, otherwise, we have to keep the default values the sync
 with the backend functions. The benefit of having the default values in the
 payloads is that a user can find the default values in the Restful API docs.
 """
-import getpass
 import os
 import typing
 from typing import Any, Dict, List, Optional, Tuple, Union
@@ -58,8 +57,7 @@ def request_body_env_vars() -> dict:
         if common.is_api_server_local() and env_var in EXTERNAL_LOCAL_ENV_VARS:
             env_vars[env_var] = os.environ[env_var]
     env_vars[constants.USER_ID_ENV_VAR] = common_utils.get_user_hash()
-    env_vars[constants.USER_ENV_VAR] = os.getenv(constants.USER_ENV_VAR,
-                                                 getpass.getuser())
+    env_vars[constants.USER_ENV_VAR] = common_utils.get_current_user_name()
     env_vars[
         usage_constants.USAGE_RUN_ID_ENV_VAR] = usage_lib.messages.usage.run_id
     # Remove the path to config file, as the config content is included in the
@@ -370,6 +368,22 @@ class StorageBody(RequestBody):
     name: str
 
 
+class VolumeApplyBody(RequestBody):
+    """The request body for the volume apply endpoint."""
+    name: str
+    volume_type: str
+    cloud: str
+    region: Optional[str] = None
+    zone: Optional[str] = None
+    size: Optional[str] = None
+    config: Optional[Dict[str, Any]] = None
+
+
+class VolumeDeleteBody(RequestBody):
+    """The request body for the volume delete endpoint."""
+    names: List[str]
+
+
 class EndpointsBody(RequestBody):
     """The request body for the endpoint."""
     cluster: str
@@ -613,3 +627,8 @@ class UpdateConfigBody(RequestBody):
 class GetConfigBody(RequestBody):
     """The request body for getting the entire SkyPilot configuration."""
     pass
+
+
+class CostReportBody(RequestBody):
+    """The request body for the cost report endpoint."""
+    days: Optional[int] = 30

sky/server/requests/requests.py

@@ -38,6 +38,7 @@ REQUEST_TABLE = 'requests'
 COL_CLUSTER_NAME = 'cluster_name'
 COL_USER_ID = 'user_id'
 COL_STATUS_MSG = 'status_msg'
+COL_SHOULD_RETRY = 'should_retry'
 REQUEST_LOG_PATH_PREFIX = '~/sky_logs/api_server/requests'
 
 # TODO(zhwu): For scalability, there are several TODOs:
@@ -86,6 +87,7 @@ REQUEST_COLUMNS = [
     'schedule_type',
     COL_USER_ID,
     COL_STATUS_MSG,
+    COL_SHOULD_RETRY,
 ]
 
 
@@ -115,6 +117,7 @@ class RequestPayload:
     # Resources the request operates on.
     cluster_name: Optional[str] = None
     status_msg: Optional[str] = None
+    should_retry: bool = False
 
 
 @dataclasses.dataclass
@@ -137,6 +140,8 @@ class Request:
     cluster_name: Optional[str] = None
     # Status message of the request, indicates the reason of current status.
     status_msg: Optional[str] = None
+    # Whether the request should be retried.
+    should_retry: bool = False
 
     @property
     def log_path(self) -> pathlib.Path:
@@ -222,6 +227,7 @@ class Request:
             user_name=user_name,
             cluster_name=self.cluster_name,
             status_msg=self.status_msg,
+            should_retry=self.should_retry,
         )
 
     def encode(self) -> RequestPayload:
@@ -243,6 +249,7 @@ class Request:
                 user_id=self.user_id,
                 cluster_name=self.cluster_name,
                 status_msg=self.status_msg,
+                should_retry=self.should_retry,
             )
         except (TypeError, ValueError) as e:
             # The error is unexpected, so we don't suppress the stack trace.
@@ -274,6 +281,7 @@ class Request:
                 user_id=payload.user_id,
                 cluster_name=payload.cluster_name,
                 status_msg=payload.status_msg,
+                should_retry=payload.should_retry,
             )
         except (TypeError, ValueError) as e:
             logger.error(
@@ -327,6 +335,44 @@ def refresh_cluster_status_event():
         time.sleep(server_constants.CLUSTER_REFRESH_DAEMON_INTERVAL_SECONDS)
 
 
+def refresh_volume_status_event():
+    """Periodically refresh the volume status."""
+    # pylint: disable=import-outside-toplevel
+    from sky.volumes.server import core
+
+    # Disable logging for periodic refresh to avoid the usage message being
+    # sent multiple times.
+    os.environ[env_options.Options.DISABLE_LOGGING.env_key] = '1'
+
+    while True:
+        logger.info('=== Refreshing volume status ===')
+        core.volume_refresh()
+        logger.info('Volume status refreshed. Sleeping '
+                    f'{server_constants.VOLUME_REFRESH_DAEMON_INTERVAL_SECONDS}'
+                    ' seconds for the next refresh...\n')
+        time.sleep(server_constants.VOLUME_REFRESH_DAEMON_INTERVAL_SECONDS)
+
+
+def managed_job_status_refresh_event():
+    """Refresh the managed job status for controller consolidation mode."""
+    # pylint: disable=import-outside-toplevel
+    from sky.jobs import utils as managed_job_utils
+    if not managed_job_utils.is_consolidation_mode():
+        return
+    # We run the recovery logic before starting the event loop as those two are
+    # conflicting. Check PERSISTENT_RUN_RESTARTING_SIGNAL_FILE for details.
+    from sky.utils import controller_utils
+    if controller_utils.high_availability_specified(
+            controller_utils.Controllers.JOBS_CONTROLLER.value.cluster_name):
+        managed_job_utils.ha_recovery_for_consolidation_mode()
+    # After recovery, we start the event loop.
+    from sky.skylet import events
+    event = events.ManagedJobEvent()
+    while True:
+        time.sleep(events.EVENT_CHECKING_INTERVAL_SECONDS)
+        event.run()
+
+
 @dataclasses.dataclass
 class InternalRequestDaemon:
     id: str
@@ -341,7 +387,14 @@ INTERNAL_REQUEST_DAEMONS = [
     # cluster being stopped or down when `sky status -r` is called.
     InternalRequestDaemon(id='skypilot-status-refresh-daemon',
                           name='status',
-                          event_fn=refresh_cluster_status_event)
+                          event_fn=refresh_cluster_status_event),
+    # Volume status refresh daemon to update the volume status periodically.
+    InternalRequestDaemon(id='skypilot-volume-status-refresh-daemon',
+                          name='volume',
+                          event_fn=refresh_volume_status_event),
+    InternalRequestDaemon(id='managed-job-status-refresh-daemon',
+                          name='managed-job-status',
+                          event_fn=managed_job_status_refresh_event),
 ]
 
 
@@ -423,10 +476,14 @@ def create_table(cursor, conn):
         {COL_CLUSTER_NAME} TEXT,
         schedule_type TEXT,
         {COL_USER_ID} TEXT,
-        {COL_STATUS_MSG} TEXT)""")
+        {COL_STATUS_MSG} TEXT,
+        {COL_SHOULD_RETRY} INTEGER
+        )""")
 
     db_utils.add_column_to_table(cursor, conn, REQUEST_TABLE, COL_STATUS_MSG,
                                  'TEXT')
+    db_utils.add_column_to_table(cursor, conn, REQUEST_TABLE, COL_SHOULD_RETRY,
+                                 'INTEGER')
 
 
 _DB = None

sky/server/rest.py

@@ -0,0 +1,152 @@
+"""REST API client of SkyPilot API server"""
+
+import contextlib
+import contextvars
+import functools
+import time
+import typing
+from typing import Any, Callable, cast, Optional, TypeVar
+
+import colorama
+
+from sky import exceptions
+from sky import sky_logging
+from sky.adaptors import common as adaptors_common
+from sky.utils import common_utils
+from sky.utils import rich_utils
+from sky.utils import ux_utils
+
+logger = sky_logging.init_logger(__name__)
+
+if typing.TYPE_CHECKING:
+    import requests
+
+else:
+    requests = adaptors_common.LazyImport('requests')
+
+F = TypeVar('F', bound=Callable[..., Any])
+
+_RETRY_CONTEXT = contextvars.ContextVar('retry_context', default=None)
+
+
+class RetryContext:
+
+    def __init__(self):
+        self.line_processed = 0
+
+
+def retry_on_server_unavailable(max_wait_seconds: int = 600,
+                                initial_backoff: float = 5.0,
+                                max_backoff_factor: int = 5):
+    """Decorator that retries a function when ServerTemporarilyUnavailableError
+    is caught.
+
+    Args:
+        max_wait_seconds: Maximum number of seconds to wait for the server to
+            be healthy
+        initial_backoff: Initial backoff time in seconds
+        max_backoff_factor: Maximum backoff factor for exponential backoff
+
+    Notes(dev):
+        This decorator is mainly used in two scenarios:
+        1. Decorate a Restful API call to make the API call wait for server
+           recovery when server is temporarily unavailable. APIs like /api/get
+           and /api/stream should not be retried since sending them to a new
+           replica of API server will not work.
+        2. Decorate a SDK function to make the entire SDK function call get
+           retried when /api/get or /logs raises a retryable error. This
+           is typically triggered by a graceful upgrade of the API server,
+           where the pending requests and logs requests will be interrupted.
+    """
+
+    def decorator(func: F) -> F:
+
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs) -> Any:
+            msg = (
+                f'{colorama.Fore.YELLOW}API server is temporarily: upgrade in '
+                f'progress. Waiting to resume...{colorama.Style.RESET_ALL}')
+            backoff = common_utils.Backoff(
+                initial_backoff=initial_backoff,
+                max_backoff_factor=max_backoff_factor)
+            start_time = time.time()
+            attempt = 0
+
+            with _retry_in_context():
+                while True:
+                    attempt += 1
+                    try:
+                        return func(*args, **kwargs)
+                    except exceptions.ServerTemporarilyUnavailableError as e:
+                        # This will cause the status spinner being stopped and
+                        # restarted in every retry loop. But it is necessary to
+                        # stop the status spinner before retrying func() to
+                        # avoid the status spinner get stuck if the func() runs
+                        # for a long time without update status, e.g. sky logs.
+                        with rich_utils.client_status(msg):
+                            if time.time() - start_time > max_wait_seconds:
+                                # pylint: disable=line-too-long
+                                raise exceptions.ServerTemporarilyUnavailableError(
+                                    'Timeout waiting for the API server to be '
+                                    f'available after {max_wait_seconds}s.') \
+                                    from e
+
+                            sleep_time = backoff.current_backoff()
+                            time.sleep(sleep_time)
+                            logger.debug('The API server is unavailable. '
+                                         f'Retrying {func.__name__} '
+                                         f'(attempt {attempt}, '
+                                         f'backoff {sleep_time}s).')
+
+        return cast(F, wrapper)
+
+    return decorator
+
+
+@contextlib.contextmanager
+def _retry_in_context():
+    token = _RETRY_CONTEXT.set(RetryContext())
+    try:
+        yield
+    finally:
+        _RETRY_CONTEXT.reset(token)
+
+
+def get_retry_context() -> Optional[RetryContext]:
+    return _RETRY_CONTEXT.get()
+
+
+def handle_server_unavailable(response: 'requests.Response') -> None:
+    if response.status_code == 503:
+        # TODO(aylei): Hacky, depends on how nginx controller handles backends
+        # with no ready endpoints. Should use self-defined status code or header
+        # to distinguish retryable server error from general 503 errors.
+        with ux_utils.print_exception_no_traceback():
+            raise exceptions.ServerTemporarilyUnavailableError(
+                'SkyPilot API server is temporarily unavailable. '
+                'Please try again later.')
+
+
+@retry_on_server_unavailable()
+def post(url, data=None, json=None, **kwargs) -> 'requests.Response':
+    """Send a POST request to the API server, retry on server temporarily
+    unavailable."""
+    response = requests.post(url, data=data, json=json, **kwargs)
+    handle_server_unavailable(response)
+    return response
+
+
+@retry_on_server_unavailable()
+def get(url, params=None, **kwargs) -> 'requests.Response':
+    """Send a GET request to the API server, retry on server temporarily
+    unavailable."""
+    response = requests.get(url, params=params, **kwargs)
+    handle_server_unavailable(response)
+    return response
+
+
+def get_without_retry(url, params=None, **kwargs) -> 'requests.Response':
+    """Send a GET request to the API server without retry."""
+    response = requests.get(url, params=params, **kwargs)
+    handle_server_unavailable(response)
+    return response