skypilot-nightly 1.0.0.dev20250218__tar.gz → 1.0.0.dev20250220__tar.gz

{skypilot_nightly-1.0.0.dev20250218/skypilot_nightly.egg-info → skypilot_nightly-1.0.0.dev20250220}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: skypilot-nightly
-Version: 1.0.0.dev20250218
+Version: 1.0.0.dev20250220
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0

{skypilot_nightly-1.0.0.dev20250218 → skypilot_nightly-1.0.0.dev20250220}/sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '912b8293b3ebeba84941c108dbede1e6dcbc9b6f'
+_SKYPILOT_COMMIT_SHA = '6b2b31d8358f3ff8394a7a33ec49e9985ada230f'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250218'
+__version__ = '1.0.0.dev20250220'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

{skypilot_nightly-1.0.0.dev20250218 → skypilot_nightly-1.0.0.dev20250220}/sky/cli.py

@@ -1419,16 +1419,16 @@ def _handle_jobs_queue_request(
         try:
             # Check the controller status again, as the RuntimeError is likely
             # due to the controller being autostopped when querying the jobs.
-            controller_type = controller_utils.Controllers.JOBS_CONTROLLER
-            # Query status of the controller cluster. We add a wildcard because
-            # the controller cluster name can have a suffix like
-            # '-remote-<hash>' when using remote API server.
+            # Since we are client-side, we may not know the exact name of the
+            # controller, so use the prefix with a wildcard.
+            # Query status of the controller cluster.
             records = sdk.get(
-                sdk.status(
-                    cluster_names=[controller_type.value.cluster_name + '*']))
+                sdk.status(cluster_names=[common.JOB_CONTROLLER_PREFIX + '*'],
+                           all_users=True))
             if (not records or
                     records[0]['status'] == status_lib.ClusterStatus.STOPPED):
-                msg = controller_type.value.default_hint_if_non_existent
+                controller = controller_utils.Controllers.JOBS_CONTROLLER.value
+                msg = controller.default_hint_if_non_existent
         except Exception:  # pylint: disable=broad-except
             # This is to an best effort to find the latest controller status to
             # print more helpful message, so we can ignore any exception to
@@ -1494,16 +1494,18 @@ def _handle_services_request(
             # Check the controller status again, as the RuntimeError is likely
             # due to the controller being autostopped when querying the
             # services.
-            controller_type = controller_utils.Controllers.SKY_SERVE_CONTROLLER
-            # Query status of the controller cluster. We add a wildcard because
-            # the controller cluster name can have a suffix like
-            # '-remote-<hash>' when using remote API server.
+            # Since we are client-side, we may not know the exact name of the
+            # controller, so use the prefix with a wildcard.
+            # Query status of the controller cluster.
             records = sdk.get(
                 sdk.status(
-                    cluster_names=[controller_type.value.cluster_name + '*']))
+                    cluster_names=[common.SKY_SERVE_CONTROLLER_PREFIX + '*'],
+                    all_users=True))
             if (not records or
                     records[0]['status'] == status_lib.ClusterStatus.STOPPED):
-                msg = controller_type.value.default_hint_if_non_existent
+                controller = (
+                    controller_utils.Controllers.SKY_SERVE_CONTROLLER.value)
+                msg = controller.default_hint_if_non_existent
         except Exception:  # pylint: disable=broad-except
             # This is to an best effort to find the latest controller status to
             # print more helpful message, so we can ignore any exception to
@@ -2804,11 +2806,6 @@ def _hint_or_raise_for_down_jobs_controller(controller_name: str,
             to be torn down (e.g., because it has jobs running or
             it is in init state)
     """
-    if not common.is_current_user_controller(controller_name):
-        with ux_utils.print_exception_no_traceback():
-            raise exceptions.NotSupportedError(
-                f'Tearing down other user\'s managed job controller '
-                f'{controller_name!r} is not allowed.')
     controller = controller_utils.Controllers.from_name(controller_name)
     assert controller is not None, controller_name
 
@@ -2868,12 +2865,6 @@ def _hint_or_raise_for_down_sky_serve_controller(controller_name: str,
             to be torn down (e.g., because it has services running or
             it is in init state)
     """
-    # TODO(zhwu): Move this check to the sdk or even API server side.
-    if not common.is_current_user_controller(controller_name):
-        with ux_utils.print_exception_no_traceback():
-            raise exceptions.NotSupportedError(
-                f'Tearing down other user\'s sky serve controller '
-                f'{controller_name!r} is not allowed.')
     controller = controller_utils.Controllers.from_name(controller_name)
     assert controller is not None, controller_name
     with rich_utils.client_status('[bold cyan]Checking for live services[/]'):

{skypilot_nightly-1.0.0.dev20250218 → skypilot_nightly-1.0.0.dev20250220}/sky/client/cli.py

@@ -1419,16 +1419,16 @@ def _handle_jobs_queue_request(
         try:
             # Check the controller status again, as the RuntimeError is likely
             # due to the controller being autostopped when querying the jobs.
-            controller_type = controller_utils.Controllers.JOBS_CONTROLLER
-            # Query status of the controller cluster. We add a wildcard because
-            # the controller cluster name can have a suffix like
-            # '-remote-<hash>' when using remote API server.
+            # Since we are client-side, we may not know the exact name of the
+            # controller, so use the prefix with a wildcard.
+            # Query status of the controller cluster.
             records = sdk.get(
-                sdk.status(
-                    cluster_names=[controller_type.value.cluster_name + '*']))
+                sdk.status(cluster_names=[common.JOB_CONTROLLER_PREFIX + '*'],
+                           all_users=True))
             if (not records or
                     records[0]['status'] == status_lib.ClusterStatus.STOPPED):
-                msg = controller_type.value.default_hint_if_non_existent
+                controller = controller_utils.Controllers.JOBS_CONTROLLER.value
+                msg = controller.default_hint_if_non_existent
         except Exception:  # pylint: disable=broad-except
             # This is to an best effort to find the latest controller status to
             # print more helpful message, so we can ignore any exception to
@@ -1494,16 +1494,18 @@ def _handle_services_request(
             # Check the controller status again, as the RuntimeError is likely
             # due to the controller being autostopped when querying the
             # services.
-            controller_type = controller_utils.Controllers.SKY_SERVE_CONTROLLER
-            # Query status of the controller cluster. We add a wildcard because
-            # the controller cluster name can have a suffix like
-            # '-remote-<hash>' when using remote API server.
+            # Since we are client-side, we may not know the exact name of the
+            # controller, so use the prefix with a wildcard.
+            # Query status of the controller cluster.
             records = sdk.get(
                 sdk.status(
-                    cluster_names=[controller_type.value.cluster_name + '*']))
+                    cluster_names=[common.SKY_SERVE_CONTROLLER_PREFIX + '*'],
+                    all_users=True))
             if (not records or
                     records[0]['status'] == status_lib.ClusterStatus.STOPPED):
-                msg = controller_type.value.default_hint_if_non_existent
+                controller = (
+                    controller_utils.Controllers.SKY_SERVE_CONTROLLER.value)
+                msg = controller.default_hint_if_non_existent
         except Exception:  # pylint: disable=broad-except
             # This is to an best effort to find the latest controller status to
             # print more helpful message, so we can ignore any exception to
@@ -2804,11 +2806,6 @@ def _hint_or_raise_for_down_jobs_controller(controller_name: str,
             to be torn down (e.g., because it has jobs running or
             it is in init state)
     """
-    if not common.is_current_user_controller(controller_name):
-        with ux_utils.print_exception_no_traceback():
-            raise exceptions.NotSupportedError(
-                f'Tearing down other user\'s managed job controller '
-                f'{controller_name!r} is not allowed.')
     controller = controller_utils.Controllers.from_name(controller_name)
     assert controller is not None, controller_name
 
@@ -2868,12 +2865,6 @@ def _hint_or_raise_for_down_sky_serve_controller(controller_name: str,
             to be torn down (e.g., because it has services running or
             it is in init state)
     """
-    # TODO(zhwu): Move this check to the sdk or even API server side.
-    if not common.is_current_user_controller(controller_name):
-        with ux_utils.print_exception_no_traceback():
-            raise exceptions.NotSupportedError(
-                f'Tearing down other user\'s sky serve controller '
-                f'{controller_name!r} is not allowed.')
     controller = controller_utils.Controllers.from_name(controller_name)
     assert controller is not None, controller_name
     with rich_utils.client_status('[bold cyan]Checking for live services[/]'):

{skypilot_nightly-1.0.0.dev20250218 → skypilot_nightly-1.0.0.dev20250220}/sky/clouds/service_catalog/data_fetchers/fetch_gcp.py

@@ -60,8 +60,8 @@ HIDDEN_TPU_DF = pd.read_csv(
  ,tpu-v3-2048,1,,,tpu-v3-2048,2048.0,614.4,us-east1,us-east1-d
  """)))
 
-# TPU V6e price for us-central2 is missing in the SKUs.
-TPU_V6E_MISSING_REGIONS = ['us-central2']
+# TPU V6e price for the following regions is missing in the SKUs.
+TPU_V6E_MISSING_REGIONS = ['us-central2', 'southamerica-west1']
 
 # TPU V5 is not visible in specific zones. We hardcode the missing zones here.
 # NOTE(dev): Keep the zones and the df in sync.

{skypilot_nightly-1.0.0.dev20250218 → skypilot_nightly-1.0.0.dev20250220}/sky/jobs/server/core.py

@@ -21,10 +21,11 @@ from sky.backends import backend_utils
 from sky.clouds.service_catalog import common as service_catalog_common
 from sky.jobs import constants as managed_job_constants
 from sky.jobs import utils as managed_job_utils
-from sky.provision import common
+from sky.provision import common as provision_common
 from sky.skylet import constants as skylet_constants
 from sky.usage import usage_lib
 from sky.utils import admin_policy_utils
+from sky.utils import common
 from sky.utils import common_utils
 from sky.utils import controller_utils
 from sky.utils import dag_utils
@@ -149,14 +150,18 @@ def launch(
             f'{colorama.Fore.YELLOW}'
             f'Launching managed job {dag.name!r} from jobs controller...'
             f'{colorama.Style.RESET_ALL}')
-        return execution.launch(task=controller_task,
-                                cluster_name=controller_name,
-                                stream_logs=stream_logs,
-                                idle_minutes_to_autostop=skylet_constants.
-                                CONTROLLER_IDLE_MINUTES_TO_AUTOSTOP,
-                                retry_until_up=True,
-                                fast=True,
-                                _disable_controller_check=True)
+
+        # Launch with the api server's user hash, so that sky status does not
+        # show the owner of the controller as whatever user launched it first.
+        with common.with_server_user_hash():
+            return execution.launch(task=controller_task,
+                                    cluster_name=controller_name,
+                                    stream_logs=stream_logs,
+                                    idle_minutes_to_autostop=skylet_constants.
+                                    CONTROLLER_IDLE_MINUTES_TO_AUTOSTOP,
+                                    retry_until_up=True,
+                                    fast=True,
+                                    _disable_controller_check=True)
 
 
 def queue_from_kubernetes_pod(
@@ -194,16 +199,16 @@ def queue_from_kubernetes_pod(
     provider_config = {'context': context}
     instances = {
         pod_name: [
-            common.InstanceInfo(instance_id=pod_name,
-                                internal_ip='',
-                                external_ip='',
-                                tags={})
+            provision_common.InstanceInfo(instance_id=pod_name,
+                                          internal_ip='',
+                                          external_ip='',
+                                          tags={})
         ]
     }  # Internal IP is not required for Kubernetes
-    cluster_info = common.ClusterInfo(provider_name='kubernetes',
-                                      head_instance_id=pod_name,
-                                      provider_config=provider_config,
-                                      instances=instances)
+    cluster_info = provision_common.ClusterInfo(provider_name='kubernetes',
+                                                head_instance_id=pod_name,
+                                                provider_config=provider_config,
+                                                instances=instances)
     managed_jobs_runner = provision_lib.get_command_runners(
         'kubernetes', cluster_info)[0]
 

{skypilot_nightly-1.0.0.dev20250218 → skypilot_nightly-1.0.0.dev20250220}/sky/jobs/server/dashboard_utils.py

@@ -1,4 +1,9 @@
-"""Persistent dashboard sessions."""
+"""Persistent dashboard sessions.
+
+Note: before #4717, this was useful because we needed to tunnel to multiple
+controllers - one per user. Now, there is only one controller for the whole API
+server, so this is not very useful. TODO(cooperc): Remove or fix this.
+"""
 import pathlib
 from typing import Tuple
 

{skypilot_nightly-1.0.0.dev20250218 → skypilot_nightly-1.0.0.dev20250220}/sky/jobs/server/server.py

@@ -21,11 +21,6 @@ logger = sky_logging.init_logger(__name__)
 router = fastapi.APIRouter()
 
 
-def _get_controller_name(request_body: payloads.RequestBody) -> str:
-    user_hash = request_body.user_hash
-    return common.get_controller_name(common.ControllerType.JOBS, user_hash)
-
-
 @router.post('/launch')
 async def launch(request: fastapi.Request,
                  jobs_launch_body: payloads.JobsLaunchBody) -> None:
@@ -35,7 +30,7 @@ async def launch(request: fastapi.Request,
         request_body=jobs_launch_body,
         func=core.launch,
         schedule_type=api_requests.ScheduleType.LONG,
-        request_cluster_name=_get_controller_name(jobs_launch_body),
+        request_cluster_name=common.JOB_CONTROLLER_NAME,
     )
 
 
@@ -49,7 +44,7 @@ async def queue(request: fastapi.Request,
         func=core.queue,
         schedule_type=(api_requests.ScheduleType.LONG if jobs_queue_body.refresh
                        else api_requests.ScheduleType.SHORT),
-        request_cluster_name=_get_controller_name(jobs_queue_body),
+        request_cluster_name=common.JOB_CONTROLLER_NAME,
     )
 
 
@@ -62,7 +57,7 @@ async def cancel(request: fastapi.Request,
         request_body=jobs_cancel_body,
         func=core.cancel,
         schedule_type=api_requests.ScheduleType.SHORT,
-        request_cluster_name=_get_controller_name(jobs_cancel_body),
+        request_cluster_name=common.JOB_CONTROLLER_NAME,
     )
 
 
@@ -78,7 +73,7 @@ async def logs(
         func=core.tail_logs,
         schedule_type=api_requests.ScheduleType.SHORT
         if jobs_logs_body.refresh else api_requests.ScheduleType.LONG,
-        request_cluster_name=_get_controller_name(jobs_logs_body),
+        request_cluster_name=common.JOB_CONTROLLER_NAME,
     )
     request_task = api_requests.get_request(request.state.request_id)
 
@@ -107,13 +102,16 @@ async def download_logs(
         func=core.download_logs,
         schedule_type=api_requests.ScheduleType.LONG
         if jobs_download_logs_body.refresh else api_requests.ScheduleType.SHORT,
-        request_cluster_name=_get_controller_name(jobs_download_logs_body),
+        request_cluster_name=common.JOB_CONTROLLER_NAME,
     )
 
 
 @router.get('/dashboard')
 async def dashboard(request: fastapi.Request,
                     user_hash: str) -> fastapi.Response:
+    # Note: before #4717, each user had their own controller, and thus their own
+    # dashboard. Now, all users share the same controller, so this isn't really
+    # necessary. TODO(cooperc): clean up.
     # Find the port for the dashboard of the user
     os.environ[constants.USER_ID_ENV_VAR] = user_hash
     server_common.reload_for_new_request(client_entrypoint=None,

{skypilot_nightly-1.0.0.dev20250218 → skypilot_nightly-1.0.0.dev20250220}/sky/serve/server/core.py

@@ -249,13 +249,16 @@ def up(
         # with the current job id, we know the service is up and running
         # for the first time; otherwise it is a name conflict.
         idle_minutes_to_autostop = constants.CONTROLLER_IDLE_MINUTES_TO_AUTOSTOP
-        controller_job_id, controller_handle = execution.launch(
-            task=controller_task,
-            cluster_name=controller_name,
-            idle_minutes_to_autostop=idle_minutes_to_autostop,
-            retry_until_up=True,
-            _disable_controller_check=True,
-        )
+        # Since the controller may be shared among multiple users, launch the
+        # controller with the API server's user hash.
+        with common.with_server_user_hash():
+            controller_job_id, controller_handle = execution.launch(
+                task=controller_task,
+                cluster_name=controller_name,
+                idle_minutes_to_autostop=idle_minutes_to_autostop,
+                retry_until_up=True,
+                _disable_controller_check=True,
+            )
 
         style = colorama.Style
         fore = colorama.Fore

{skypilot_nightly-1.0.0.dev20250218 → skypilot_nightly-1.0.0.dev20250220}/sky/serve/server/server.py

@@ -14,11 +14,6 @@ logger = sky_logging.init_logger(__name__)
 router = fastapi.APIRouter()
 
 
-def _get_controller_name(request_body: payloads.RequestBody) -> str:
-    user_hash = request_body.user_hash
-    return common.get_controller_name(common.ControllerType.SERVE, user_hash)
-
-
 @router.post('/up')
 async def up(
     request: fastapi.Request,
@@ -30,7 +25,7 @@ async def up(
         request_body=up_body,
         func=core.up,
         schedule_type=api_requests.ScheduleType.LONG,
-        request_cluster_name=_get_controller_name(up_body),
+        request_cluster_name=common.SKY_SERVE_CONTROLLER_NAME,
     )
 
 
@@ -45,7 +40,7 @@ async def update(
         request_body=update_body,
         func=core.update,
         schedule_type=api_requests.ScheduleType.SHORT,
-        request_cluster_name=_get_controller_name(update_body),
+        request_cluster_name=common.SKY_SERVE_CONTROLLER_NAME,
     )
 
 
@@ -60,7 +55,7 @@ async def down(
         request_body=down_body,
         func=core.down,
         schedule_type=api_requests.ScheduleType.SHORT,
-        request_cluster_name=_get_controller_name(down_body),
+        request_cluster_name=common.SKY_SERVE_CONTROLLER_NAME,
     )
 
 
@@ -75,7 +70,7 @@ async def terminate_replica(
         request_body=terminate_replica_body,
         func=core.terminate_replica,
         schedule_type=api_requests.ScheduleType.SHORT,
-        request_cluster_name=_get_controller_name(terminate_replica_body),
+        request_cluster_name=common.SKY_SERVE_CONTROLLER_NAME,
     )
 
 
@@ -90,7 +85,7 @@ async def status(
         request_body=status_body,
         func=core.status,
         schedule_type=api_requests.ScheduleType.SHORT,
-        request_cluster_name=_get_controller_name(status_body),
+        request_cluster_name=common.SKY_SERVE_CONTROLLER_NAME,
     )
 
 
@@ -105,7 +100,7 @@ async def tail_logs(
         request_body=log_body,
         func=core.tail_logs,
         schedule_type=api_requests.ScheduleType.SHORT,
-        request_cluster_name=_get_controller_name(log_body),
+        request_cluster_name=common.SKY_SERVE_CONTROLLER_NAME,
     )
 
     request_task = api_requests.get_request(request.state.request_id)

{skypilot_nightly-1.0.0.dev20250218 → skypilot_nightly-1.0.0.dev20250220}/sky/server/common.py

@@ -3,7 +3,6 @@
 import dataclasses
 import enum
 import functools
-import importlib
 import json
 import os
 import pathlib
@@ -16,7 +15,6 @@ import uuid
 
 import colorama
 import filelock
-import psutil
 import pydantic
 import requests
 
@@ -28,14 +26,12 @@ from sky.server import constants as server_constants
 from sky.skylet import constants
 from sky.usage import usage_lib
 from sky.utils import annotations
-from sky.utils import common
 from sky.utils import common_utils
 from sky.utils import rich_utils
 from sky.utils import ux_utils
 
 if typing.TYPE_CHECKING:
     from sky import dag as dag_lib
-    from sky.server.requests import payloads
 
 DEFAULT_SERVER_URL = 'http://127.0.0.1:46580'
 AVAILBLE_LOCAL_API_SERVER_HOSTS = ['0.0.0.0', 'localhost', '127.0.0.1']
@@ -149,13 +145,14 @@ def get_api_server_status(endpoint: Optional[str] = None) -> ApiServerInfo:
     return ApiServerInfo(status=ApiServerStatus.UNHEALTHY, api_version=None)
 
 
-def start_uvicorn_in_background(deploy: bool = False, host: str = '127.0.0.1'):
+def start_api_server_in_background(deploy: bool = False,
+                                   host: str = '127.0.0.1'):
     if not is_api_server_local():
         raise RuntimeError(
             f'Cannot start API server: {get_server_url()} is not a local URL')
 
     # Check available memory before starting the server.
-    avail_mem_size_gb: float = psutil.virtual_memory().available / (1024**3)
+    avail_mem_size_gb: float = common_utils.get_mem_size_gb()
     if avail_mem_size_gb <= server_constants.MIN_AVAIL_MEM_GB:
         logger.warning(
             f'{colorama.Fore.YELLOW}Your SkyPilot API server machine only has '
@@ -166,8 +163,6 @@ def start_uvicorn_in_background(deploy: bool = False, host: str = '127.0.0.1'):
     log_path = os.path.expanduser(constants.API_SERVER_LOGS)
     os.makedirs(os.path.dirname(log_path), exist_ok=True)
 
-    # The command to run uvicorn. Adjust the app:app to your application's
-    # location.
     api_server_cmd = API_SERVER_CMD
     if deploy:
         api_server_cmd += ' --deploy'
@@ -175,7 +170,7 @@ def start_uvicorn_in_background(deploy: bool = False, host: str = '127.0.0.1'):
         api_server_cmd += f' --host {host}'
     cmd = f'{sys.executable} {api_server_cmd} > {log_path} 2>&1'
 
-    # Start the uvicorn process in the background and don't wait for it.
+    # Start the API server process in the background and don't wait for it.
     # If this is called from a CLI invocation, we need start_new_session=True so
     # that SIGINT on the CLI will not also kill the API server.
     subprocess.Popen(cmd, shell=True, start_new_session=True)
@@ -235,7 +230,7 @@ def _start_api_server(deploy: bool = False, host: str = '127.0.0.1'):
                     f'SkyPilot API server at {server_url}. '
                     'Starting a local server.'
                     f'{colorama.Style.RESET_ALL}')
-        start_uvicorn_in_background(deploy=deploy, host=host)
+        start_api_server_in_background(deploy=deploy, host=host)
         logger.info(ux_utils.finishing_message('SkyPilot API server started.'))
 
 
@@ -407,23 +402,6 @@ def request_body_to_params(body: pydantic.BaseModel) -> Dict[str, Any]:
 def reload_for_new_request(client_entrypoint: Optional[str],
                            client_command: Optional[str]):
     """Reload modules, global variables, and usage message for a new request."""
-    # When a user request is sent to api server, it changes the user hash in the
-    # env vars, but since controller_utils is imported before the env vars are
-    # set, it doesn't get updated. So we need to reload it here.
-    # pylint: disable=import-outside-toplevel
-    from sky.utils import controller_utils
-    common.SKY_SERVE_CONTROLLER_NAME = common.get_controller_name(
-        common.ControllerType.SERVE)
-    common.JOB_CONTROLLER_NAME = common.get_controller_name(
-        common.ControllerType.JOBS)
-    # TODO(zhwu): We should avoid reloading the controller_utils module.
-    # Instead, we should reload required cache or global variables.
-    # TODO(zhwu): Reloading the controller_utils module may cause the global
-    # variables in other modules referring the `controller_utils.Controllers`
-    # dangling, as they will be pointing to the old object. We should not use
-    # it in global variables.
-    importlib.reload(controller_utils)
-
     # Reset the client entrypoint and command for the usage message.
     common_utils.set_client_entrypoint_and_command(
         client_entrypoint=client_entrypoint,