skypilot-nightly 1.0.0.dev20250807__py3-none-any.whl → 1.0.0.dev20250812__py3-none-any.whl

sky/serve/serve_utils.py

@@ -37,6 +37,7 @@ from sky.skylet import job_lib
 from sky.utils import annotations
 from sky.utils import command_runner
 from sky.utils import common_utils
+from sky.utils import controller_utils
 from sky.utils import log_utils
 from sky.utils import message_utils
 from sky.utils import resources_utils
@@ -259,14 +260,47 @@ def get_service_filelock_path(pool: str) -> str:
     return str(path)
 
 
+def _validate_consolidation_mode_config(current_is_consolidation_mode: bool,
+                                        pool: bool) -> None:
+    """Validate the consolidation mode config."""
+    # Check whether the consolidation mode config is changed.
+    controller = controller_utils.get_controller_for_pool(pool).value
+    if current_is_consolidation_mode:
+        controller_cn = controller.cluster_name
+        if global_user_state.get_cluster_from_name(controller_cn) is not None:
+            with ux_utils.print_exception_no_traceback():
+                raise exceptions.InconsistentConsolidationModeError(
+                    f'{colorama.Fore.RED}Consolidation mode for '
+                    f'{controller.controller_type} is enabled, but the '
+                    f'controller cluster {controller_cn} is still running. '
+                    'Please terminate the controller cluster first.'
+                    f'{colorama.Style.RESET_ALL}')
+    else:
+        noun = 'pool' if pool else 'service'
+        all_services = [
+            svc for svc in serve_state.get_services() if svc['pool'] == pool
+        ]
+        if all_services:
+            with ux_utils.print_exception_no_traceback():
+                raise exceptions.InconsistentConsolidationModeError(
+                    f'{colorama.Fore.RED}Consolidation mode for '
+                    f'{controller.controller_type} is disabled, but there are '
+                    f'still {len(all_services)} {noun}s running. Please '
+                    f'terminate those {noun}s first.{colorama.Style.RESET_ALL}')
+
+
 @annotations.lru_cache(scope='request', maxsize=1)
 def is_consolidation_mode(pool: bool = False) -> bool:
     # Use jobs config for pool consolidation mode.
-    controller_type = 'jobs' if pool else 'serve'
+    controller = controller_utils.get_controller_for_pool(pool).value
     consolidation_mode = skypilot_config.get_nested(
-        (controller_type, 'controller', 'consolidation_mode'),
+        (controller.controller_type, 'controller', 'consolidation_mode'),
         default_value=False)
-    # _check_consolidation_mode_consistency(consolidation_mode, pool)
+    # We should only do this check on API server, as the controller will not
+    # have related config and will always seemingly disabled for consolidation
+    # mode. Check #6611 for more details.
+    if os.environ.get(skylet_constants.ENV_VAR_IS_SKYPILOT_SERVER) is not None:
+        _validate_consolidation_mode_config(consolidation_mode, pool)
     return consolidation_mode
 
 

sky/serve/server/impl.py

@@ -198,9 +198,9 @@ def up(
             # We need a unique integer per sky.serve.up call to avoid name
             # conflict. Originally in non-consolidation mode, this is the ray
             # job id; now we use the request id hash instead. Here we also
-            # make sure it is a 63-bit integer to avoid overflow on sqlalchemy.
+            # make sure it is a 32-bit integer to avoid overflow on sqlalchemy.
             rid = common_utils.get_current_request_id()
-            controller_job_id = hash(uuid.UUID(rid).int) & 0x7FFFFFFFFFFFFFFF
+            controller_job_id = hash(uuid.UUID(rid).int) & 0x7FFFFFFF
 
         vars_to_fill = {
             'remote_task_yaml_path': remote_tmp_task_yaml_path,

sky/server/common.py

@@ -16,8 +16,8 @@ import tempfile
 import threading
 import time
 import typing
-from typing import (Any, Callable, cast, Dict, Literal, Optional, Tuple,
-                    TypeVar, Union)
+from typing import (Any, Callable, cast, Dict, Generic, Literal, Optional,
+                    Tuple, TypeVar, Union)
 from urllib import parse
 import uuid
 
@@ -89,16 +89,20 @@ _SERVER_INSTALL_VERSION_MISMATCH_WARNING = (
     'restarting the API server.'
     f'{colorama.Style.RESET_ALL}')
 
-RequestId = str
+T = TypeVar('T')
+P = ParamSpec('P')
+
+
+class RequestId(str, Generic[T]):
+    pass
+
+
 ApiVersion = Optional[str]
 
 logger = sky_logging.init_logger(__name__)
 
 hinted_for_server_install_version_mismatch = False
 
-T = TypeVar('T')
-P = ParamSpec('P')
-
 
 class ApiServerStatus(enum.Enum):
     HEALTHY = 'healthy'
@@ -491,7 +495,7 @@ def handle_request_error(response: 'requests.Response') -> None:
                 f'{response.text}')
 
 
-def get_request_id(response: 'requests.Response') -> RequestId:
+def get_request_id(response: 'requests.Response') -> RequestId[T]:
     handle_request_error(response)
     request_id = response.headers.get('X-Skypilot-Request-ID')
     if request_id is None:
@@ -502,7 +506,7 @@ def get_request_id(response: 'requests.Response') -> RequestId:
                 'Failed to get request ID from SkyPilot API server at '
                 f'{get_server_url()}. Response: {response.status_code} '
                 f'{response.text}')
-    return request_id
+    return RequestId[T](request_id)
 
 
 def _start_api_server(deploy: bool = False,

sky/server/constants.py

@@ -10,7 +10,7 @@ from sky.skylet import constants
 # based on version info is needed.
 # For more details and code guidelines, refer to:
 # https://docs.skypilot.co/en/latest/developers/CONTRIBUTING.html#backward-compatibility-guidelines
-API_VERSION = 15
+API_VERSION = 16
 
 # The minimum peer API version that the code should still work with.
 # Notes (dev):

sky/setup_files/alembic.ini

@@ -94,6 +94,10 @@ version_table = alembic_version_state_db
 version_locations = %(here)s/../schemas/db/spot_jobs
 version_table = alembic_version_spot_jobs_db
 
+[serve_db]
+version_locations = %(here)s/../schemas/db/serve_state
+version_table = alembic_version_serve_state_db
+
 [post_write_hooks]
 # post_write_hooks defines scripts or Python functions that are run
 # on newly generated revision scripts.  See the documentation for further

sky/skypilot_config.py

@@ -575,8 +575,8 @@ def _reload_config_as_server() -> None:
         with _DB_USE_LOCK:
             sqlalchemy_engine = sqlalchemy.create_engine(db_url,
                                                          poolclass=NullPool)
-            db_utils.add_tables_to_db_sqlalchemy(Base.metadata,
-                                                 sqlalchemy_engine)
+            db_utils.add_all_tables_to_db_sqlalchemy(Base.metadata,
+                                                     sqlalchemy_engine)
 
             def _get_config_yaml_from_db(
                     key: str) -> Optional[config_utils.Config]:
@@ -867,8 +867,8 @@ def update_api_server_config_no_lock(config: config_utils.Config) -> None:
             with _DB_USE_LOCK:
                 sqlalchemy_engine = sqlalchemy.create_engine(existing_db_url,
                                                              poolclass=NullPool)
-                db_utils.add_tables_to_db_sqlalchemy(Base.metadata,
-                                                     sqlalchemy_engine)
+                db_utils.add_all_tables_to_db_sqlalchemy(
+                    Base.metadata, sqlalchemy_engine)
 
                 def _set_config_yaml_to_db(key: str,
                                            config: config_utils.Config):

sky/users/permission.py

@@ -44,7 +44,7 @@ class PermissionService:
             if _enforcer_instance is None:
                 _enforcer_instance = self
                 engine = global_user_state.initialize_and_get_db()
-                db_utils.add_tables_to_db_sqlalchemy(
+                db_utils.add_all_tables_to_db_sqlalchemy(
                     sqlalchemy_adapter.Base.metadata, engine)
                 adapter = sqlalchemy_adapter.Adapter(engine)
                 model_path = os.path.join(os.path.dirname(__file__),

sky/utils/cli_utils/status_utils.py

@@ -81,6 +81,7 @@ def show_status_table(cluster_records: List[_ClusterRecord],
                          _get_command,
                          truncate=not show_all,
                          show_by_default=False),
+            StatusColumn('LAST_EVENT', _get_last_event, show_by_default=False),
         ]
 
     columns = []
@@ -314,6 +315,14 @@ def _get_head_ip(cluster_record: _ClusterRecord, truncate: bool = True) -> str:
     return handle.head_ip
 
 
+def _get_last_event(cluster_record: _ClusterRecord,
+                    truncate: bool = True) -> str:
+    del truncate
+    if cluster_record.get('last_event', None) is None:
+        return 'No recorded events.'
+    return cluster_record['last_event']
+
+
 def _is_pending_autostop(cluster_record: _ClusterRecord) -> bool:
     # autostop < 0 means nothing scheduled.
     return cluster_record['autostop'] >= 0 and _get_status(
@@ -401,7 +410,7 @@ def _get_estimated_cost_for_cost_report(
 
 
 def show_kubernetes_cluster_status_table(
-        clusters: List['kubernetes_utils.KubernetesSkyPilotClusterInfo'],
+        clusters: List['kubernetes_utils.KubernetesSkyPilotClusterInfoPayload'],
         show_all: bool) -> None:
     """Compute cluster table values and display for Kubernetes clusters."""
     status_columns = [

sky/utils/db/db_utils.py

@@ -87,7 +87,7 @@ def add_column_to_table(
     conn.commit()
 
 
-def add_tables_to_db_sqlalchemy(
+def add_all_tables_to_db_sqlalchemy(
     metadata: sqlalchemy.MetaData,
     engine: sqlalchemy.Engine,
 ):
@@ -103,6 +103,27 @@ def add_tables_to_db_sqlalchemy(
                 raise
 
 
+def add_table_to_db_sqlalchemy(
+    metadata: sqlalchemy.MetaData,
+    engine: sqlalchemy.Engine,
+    table_name: str,
+):
+    """Add a specific table to the database."""
+    try:
+        table = metadata.tables[table_name]
+    except KeyError as e:
+        raise e
+
+    try:
+        table.create(bind=engine, checkfirst=True)
+    except (sqlalchemy_exc.OperationalError,
+            sqlalchemy_exc.ProgrammingError) as e:
+        if 'already exists' in str(e):
+            pass
+        else:
+            raise
+
+
 def add_column_to_table_sqlalchemy(
     session: 'Session',
     table_name: str,
@@ -205,6 +226,37 @@ def add_column_to_table_alembic(
             raise
 
 
+def drop_column_from_table_alembic(
+    table_name: str,
+    column_name: str,
+):
+    """Drop a column from a table using Alembic operations.
+
+    Args:
+        table_name: Name of the table to drop column from.
+        column_name: Name of the column to drop.
+    """
+    from alembic import op  # pylint: disable=import-outside-toplevel
+
+    # Check if column exists before trying to drop it
+    bind = op.get_bind()
+    inspector = sqlalchemy.inspect(bind)
+    columns = [col['name'] for col in inspector.get_columns(table_name)]
+
+    if column_name not in columns:
+        # Column doesn't exist; nothing to do
+        return
+
+    try:
+        op.drop_column(table_name, column_name)
+    except (sqlalchemy_exc.ProgrammingError,
+            sqlalchemy_exc.OperationalError) as e:
+        if 'does not exist' in str(e).lower():
+            pass  # Already dropped
+        else:
+            raise
+
+
 class SQLiteConn(threading.local):
     """Thread-local connection to the sqlite3 database."""
 

sky/utils/db/migration_utils.py

@@ -19,13 +19,17 @@ logger = sky_logging.init_logger(__name__)
 DB_INIT_LOCK_TIMEOUT_SECONDS = 10
 
 GLOBAL_USER_STATE_DB_NAME = 'state_db'
-GLOBAL_USER_STATE_VERSION = '002'
+GLOBAL_USER_STATE_VERSION = '005'
 GLOBAL_USER_STATE_LOCK_PATH = '~/.sky/locks/.state_db.lock'
 
 SPOT_JOBS_DB_NAME = 'spot_jobs_db'
 SPOT_JOBS_VERSION = '003'
 SPOT_JOBS_LOCK_PATH = '~/.sky/locks/.spot_jobs_db.lock'
 
+SERVE_DB_NAME = 'serve_db'
+SERVE_VERSION = '001'
+SERVE_LOCK_PATH = '~/.sky/locks/.serve_db.lock'
+
 
 def get_engine(db_name: str):
     conn_string = None

sky/utils/kubernetes/deploy_remote_cluster.py

@@ -1276,7 +1276,9 @@ def deploy_cluster(head_node,
     print(
         '  • Launch a GPU development pod: sky launch -c devbox --cloud kubernetes'
     )
-    print('  • Connect to pod with VSCode: code --remote ssh-remote+devbox ')
+    print(
+        '  • Connect to pod with VSCode: code --remote ssh-remote+devbox "/home"'
+    )
     # Print completion marker for current cluster
     print(f'{GREEN}SKYPILOT_CLUSTER_COMPLETED: {NC}')
 

sky/utils/resource_checker.py

@@ -74,27 +74,7 @@ def _check_active_resources(resource_operations: List[Tuple[str, str]],
         ValueError: If any resource has active clusters or managed jobs.
     """
 
-    def get_all_clusters():
-        return global_user_state.get_clusters()
-
-    def get_all_managed_jobs():
-        # pylint: disable=import-outside-toplevel
-        from sky.jobs.server import core as managed_jobs_core
-        try:
-            return managed_jobs_core.queue(refresh=False,
-                                           skip_finished=True,
-                                           all_users=True)
-        except exceptions.ClusterNotUpError:
-            logger.warning('All jobs should be finished.')
-            return []
-
-    # Fetch both clusters and jobs in parallel
-    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
-        clusters_future = executor.submit(get_all_clusters)
-        jobs_future = executor.submit(get_all_managed_jobs)
-
-        all_clusters = clusters_future.result()
-        all_managed_jobs = jobs_future.result()
+    all_clusters, all_managed_jobs = _get_active_resources()
 
     # Collect all error messages instead of raising immediately
     error_messages = []
@@ -134,6 +114,11 @@ def _check_active_resources(resource_operations: List[Tuple[str, str]],
         # If this resource has issues, add to overall error messages
         if resource_errors:
             resource_error_summary = ' and '.join(resource_errors)
+            if resource_type == 'user':
+                # resource_name is user_id
+                user_info = global_user_state.get_user(resource_name)
+                if user_info and user_info.name:
+                    resource_name = user_info.name
             error_messages.append(
                 f'Cannot {operation} {resource_type} {resource_name!r} '
                 f'because it has {resource_error_summary}.')
@@ -151,3 +136,159 @@ def _check_active_resources(resource_operations: List[Tuple[str, str]],
                             '\n'.join(f'• {msg}' for msg in error_messages) +
                             '\nPlease terminate these resources first.')
         raise ValueError(full_message)
+
+
+def check_users_workspaces_active_resources(
+        user_ids: List[str],
+        workspace_names: List[str]) -> Tuple[str, List[str]]:
+    """Check if all the active clusters or managed jobs in workspaces
+       belong to the user_ids. If not, return the error message.
+
+    Args:
+        user_ids: List of user_id.
+        workspace_names: List of workspace_name.
+
+    Returns:
+        resource_error_summary: str
+        missed_users_names: List[str]
+    """
+    all_clusters, all_managed_jobs = _get_active_resources_for_workspaces(
+        workspace_names)
+    resource_errors = []
+    missed_users = set()
+    active_cluster_names = []
+    active_job_names = []
+    # Check clusters
+    if all_clusters:
+        for cluster in all_clusters:
+            user_hash = cluster.get('user_hash')
+            if user_hash and user_hash not in user_ids:
+                missed_users.add(user_hash)
+                active_cluster_names.append(cluster['name'])
+        if active_cluster_names:
+            cluster_list = ', '.join(active_cluster_names)
+            resource_errors.append(
+                f'{len(active_cluster_names)} active cluster(s):'
+                f' {cluster_list}')
+
+    # Check managed jobs
+    if all_managed_jobs:
+        for job in all_managed_jobs:
+            user_hash = job.get('user_hash')
+            if user_hash and user_hash not in user_ids:
+                missed_users.add(user_hash)
+                active_job_names.append(str(job['job_id']))
+        if active_job_names:
+            job_list = ', '.join(active_job_names)
+            resource_errors.append(f'{len(active_job_names)} active'
+                                   f' managed job(s): {job_list}')
+
+    resource_error_summary = ''
+    if resource_errors:
+        resource_error_summary = ' and '.join(resource_errors)
+    missed_users_names = []
+    if missed_users:
+        all_users = global_user_state.get_all_users()
+        missed_users_names = [
+            user.name if user.name else user.id
+            for user in all_users
+            if user.id in missed_users
+        ]
+    return resource_error_summary, missed_users_names
+
+
+def _get_active_resources_for_workspaces(
+    workspace_names: List[str]
+) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
+    """Get active clusters or managed jobs for workspaces.
+
+    Args:
+        workspace_names: List of workspace_name.
+
+    Returns:
+        all_clusters: List[Dict[str, Any]]
+        all_managed_jobs: List[Dict[str, Any]]
+    """
+    if not workspace_names:
+        return [], []
+
+    def filter_by_workspaces(workspace_names: List[str]):
+        return lambda resource: (resource.get(
+            'workspace', constants.SKYPILOT_DEFAULT_WORKSPACE) in
+                                 workspace_names)
+
+    return _get_active_resources_by_names(workspace_names, filter_by_workspaces)
+
+
+def _get_active_resources_by_names(
+    resource_names: List[str],
+    filter_factory: Callable[[List[str]], Callable[[Dict[str, Any]], bool]]
+) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
+    """Get active clusters or managed jobs.
+
+    Args:
+        resource_names: List of resource_name.
+        filter_factory: Function that takes a resource_name and returns a filter
+            function for clusters/jobs.
+
+    Returns:
+        all_clusters: List[Dict[str, Any]]
+        all_managed_jobs: List[Dict[str, Any]]
+    """
+
+    all_clusters, all_managed_jobs = _get_active_resources()
+
+    resource_clusters = []
+    resource_active_jobs = []
+
+    # Check each resource against the fetched data,
+    # return the active resources by names
+    resource_filter = filter_factory(resource_names)
+
+    # Filter clusters for this resource
+    if all_clusters:
+        resource_clusters = [
+            cluster for cluster in all_clusters if resource_filter(cluster)
+        ]
+
+    # Filter managed jobs for this resource
+    if all_managed_jobs:
+        resource_active_jobs = [
+            job for job in all_managed_jobs if resource_filter(job)
+        ]
+
+    return resource_clusters, resource_active_jobs
+
+
+def _get_active_resources(
+) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
+    """Get all active clusters and managed jobs.
+
+    Returns:
+        all_clusters: List[Dict[str, Any]]
+        all_managed_jobs: List[Dict[str, Any]]
+    """
+
+    def get_all_clusters():
+        return global_user_state.get_clusters()
+
+    def get_all_managed_jobs():
+        # pylint: disable=import-outside-toplevel
+        from sky.jobs.server import core as managed_jobs_core
+        try:
+            return managed_jobs_core.queue(refresh=False,
+                                           skip_finished=True,
+                                           all_users=True)
+        except exceptions.ClusterNotUpError:
+            logger.warning('All jobs should be finished.')
+            return []
+
+    # Fetch both clusters and jobs in parallel
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
+        clusters_future = executor.submit(get_all_clusters)
+        jobs_future = executor.submit(get_all_managed_jobs)
+
+        all_clusters = clusters_future.result()
+        all_managed_jobs = jobs_future.result()
+
+    return all_clusters, all_managed_jobs

sky/volumes/client/sdk.py

@@ -1,7 +1,7 @@
 """SDK functions for managed jobs."""
 import json
 import typing
-from typing import List
+from typing import Any, Dict, List
 
 from sky import sky_logging
 from sky.adaptors import common as adaptors_common
@@ -24,7 +24,7 @@ logger = sky_logging.init_logger(__name__)
 @usage_lib.entrypoint
 @server_common.check_server_healthy_or_start
 @annotations.client_api
-def apply(volume: volume_lib.Volume) -> server_common.RequestId:
+def apply(volume: volume_lib.Volume) -> server_common.RequestId[None]:
     """Creates or registers a volume.
 
     Args:
@@ -50,7 +50,7 @@ def apply(volume: volume_lib.Volume) -> server_common.RequestId:
 @usage_lib.entrypoint
 @server_common.check_server_healthy_or_start
 @annotations.client_api
-def ls() -> server_common.RequestId:
+def ls() -> server_common.RequestId[List[Dict[str, Any]]]:
     """Lists all volumes.
 
     Returns:
@@ -65,7 +65,7 @@ def ls() -> server_common.RequestId:
 @usage_lib.entrypoint
 @server_common.check_server_healthy_or_start
 @annotations.client_api
-def delete(names: List[str]) -> server_common.RequestId:
+def delete(names: List[str]) -> server_common.RequestId[None]:
     """Deletes volumes.
 
     Args: