skypilot-nightly 1.0.0.dev20250806__py3-none-any.whl → 1.0.0.dev20250808__py3-none-any.whl

sky/jobs/server/server.py

@@ -1,5 +1,7 @@
 """REST API for managed jobs."""
 
+import pathlib
+
 import fastapi
 
 from sky import sky_logging
@@ -117,7 +119,7 @@ async def pool_apply(request: fastapi.Request,
         request_body=jobs_pool_apply_body,
         func=core.pool_apply,
         schedule_type=api_requests.ScheduleType.LONG,
-        request_cluster_name=common.SKY_SERVE_CONTROLLER_NAME,
+        request_cluster_name=common.JOB_CONTROLLER_NAME,
     )
 
 
@@ -130,7 +132,7 @@ async def pool_down(request: fastapi.Request,
         request_body=jobs_pool_down_body,
         func=core.pool_down,
         schedule_type=api_requests.ScheduleType.SHORT,
-        request_cluster_name=common.SKY_SERVE_CONTROLLER_NAME,
+        request_cluster_name=common.JOB_CONTROLLER_NAME,
     )
 
 
@@ -144,5 +146,52 @@ async def pool_status(
         request_body=jobs_pool_status_body,
         func=core.pool_status,
         schedule_type=api_requests.ScheduleType.SHORT,
-        request_cluster_name=common.SKY_SERVE_CONTROLLER_NAME,
+        request_cluster_name=common.JOB_CONTROLLER_NAME,
+    )
+
+
+@router.post('/pool_logs')
+async def pool_tail_logs(
+    request: fastapi.Request, log_body: payloads.JobsPoolLogsBody,
+    background_tasks: fastapi.BackgroundTasks
+) -> fastapi.responses.StreamingResponse:
+    executor.schedule_request(
+        request_id=request.state.request_id,
+        request_name='jobs.pool_logs',
+        request_body=log_body,
+        func=core.pool_tail_logs,
+        schedule_type=api_requests.ScheduleType.SHORT,
+        request_cluster_name=common.JOB_CONTROLLER_NAME,
+    )
+
+    request_task = api_requests.get_request(request.state.request_id)
+
+    return stream_utils.stream_response(
+        request_id=request_task.request_id,
+        logs_path=request_task.log_path,
+        background_tasks=background_tasks,
+    )
+
+
+@router.post('/pool_sync-down-logs')
+async def pool_download_logs(
+    request: fastapi.Request,
+    download_logs_body: payloads.JobsPoolDownloadLogsBody,
+) -> None:
+    user_hash = download_logs_body.env_vars[constants.USER_ID_ENV_VAR]
+    timestamp = sky_logging.get_run_timestamp()
+    logs_dir_on_api_server = (
+        pathlib.Path(server_common.api_server_user_logs_dir_prefix(user_hash)) /
+        'pool' / f'{download_logs_body.pool_name}_{timestamp}')
+    logs_dir_on_api_server.mkdir(parents=True, exist_ok=True)
+    # We should reuse the original request body, so that the env vars, such as
+    # user hash, are kept the same.
+    download_logs_body.local_dir = str(logs_dir_on_api_server)
+    executor.schedule_request(
+        request_id=request.state.request_id,
+        request_name='jobs.pool_sync_down_logs',
+        request_body=download_logs_body,
+        func=core.pool_sync_down_logs,
+        schedule_type=api_requests.ScheduleType.SHORT,
+        request_cluster_name=common.JOB_CONTROLLER_NAME,
     )

sky/jobs/state.py

@@ -107,6 +107,7 @@ job_info_table = sqlalchemy.Table(
     sqlalchemy.Column('job_id_on_pool_cluster',
                       sqlalchemy.Integer,
                       server_default=None),
+    sqlalchemy.Column('pool_hash', sqlalchemy.Text, server_default=None),
 )
 
 ha_recovery_script_table = sqlalchemy.Table(
@@ -225,6 +226,7 @@ def _get_jobs_dict(r: 'row.RowMapping') -> Dict[str, Any]:
         'pool': r['pool'],
         'current_cluster_name': r['current_cluster_name'],
         'job_id_on_pool_cluster': r['job_id_on_pool_cluster'],
+        'pool_hash': r['pool_hash'],
     }
 
 
@@ -462,7 +464,8 @@ def set_job_info(job_id: int, name: str, workspace: str, entrypoint: str):
 
 @_init_db
 def set_job_info_without_job_id(name: str, workspace: str, entrypoint: str,
-                                pool: Optional[str]) -> int:
+                                pool: Optional[str],
+                                pool_hash: Optional[str]) -> int:
     assert _SQLALCHEMY_ENGINE is not None
     with orm.Session(_SQLALCHEMY_ENGINE) as session:
         if (_SQLALCHEMY_ENGINE.dialect.name ==
@@ -480,6 +483,7 @@ def set_job_info_without_job_id(name: str, workspace: str, entrypoint: str,
             workspace=workspace,
             entrypoint=entrypoint,
             pool=pool,
+            pool_hash=pool_hash,
         )
 
         if (_SQLALCHEMY_ENGINE.dialect.name ==

sky/schemas/db/global_user_state/002_add_workspace_to_cluster_history.py

@@ -0,0 +1,35 @@
+"""add workspace column to cluster_history table
+
+Revision ID: 002
+Revises: 001
+Create Date: 2025-08-06
+
+"""
+# pylint: disable=invalid-name
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+from sky.utils.db import db_utils
+
+# revision identifiers, used by Alembic.
+revision: str = '002'
+down_revision: Union[str, Sequence[str], None] = '001'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    with op.get_context().autocommit_block():
+        db_utils.add_column_to_table_alembic('cluster_history',
+                                             'workspace',
+                                             sa.Text(),
+                                             server_default=None)
+    pass
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    pass

sky/schemas/db/global_user_state/003_fix_initial_revision.py

@@ -0,0 +1,61 @@
+"""fix initial revision
+
+Revision ID: 003
+Revises: 002
+Create Date: 2025-08-07
+
+"""
+# pylint: disable=invalid-name
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+from sky.utils.db import db_utils
+
+# revision identifiers, used by Alembic.
+revision: str = '003'
+down_revision: Union[str, Sequence[str], None] = '002'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    with op.get_context().autocommit_block():
+        # add missing columns to clusters table
+        db_utils.add_column_to_table_alembic('clusters',
+                                             'storage_mounts_metadata',
+                                             sa.LargeBinary(),
+                                             server_default=None)
+        # Set the value to replace existing entries to 1 so that all the
+        # existing clusters before #2977 are considered as ever up, i.e:
+        #   existing cluster's default (null) -> 1;
+        #   new cluster's default -> 0;
+        # This is conservative for the existing clusters: even if some INIT
+        # clusters were never really UP, setting it to 1 means they won't be
+        # auto-deleted during any failover.
+        db_utils.add_column_to_table_alembic(
+            'clusters',
+            'cluster_ever_up',
+            sa.Integer(),
+            server_default='0',
+            value_to_replace_existing_entries=1)
+        db_utils.add_column_to_table_alembic('clusters',
+                                             'status_updated_at',
+                                             sa.Integer(),
+                                             server_default=None)
+
+        # remove mistakenly added columns
+        db_utils.drop_column_from_table_alembic('clusters', 'launched_nodes')
+        db_utils.drop_column_from_table_alembic('clusters', 'disk_tier')
+        db_utils.drop_column_from_table_alembic('clusters',
+                                                'config_hash_locked')
+        db_utils.drop_column_from_table_alembic('clusters', 'handle_locked')
+        db_utils.drop_column_from_table_alembic('clusters', 'num_failures')
+        db_utils.drop_column_from_table_alembic('clusters', 'configs')
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    pass

sky/schemas/db/global_user_state/004_is_managed.py

@@ -0,0 +1,34 @@
+"""Columns for whether the cluster is managed.
+
+Revision ID: 004
+Revises: 003
+Create Date: 2025-08-07
+
+"""
+# pylint: disable=invalid-name
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+from sky.utils.db import db_utils
+
+# revision identifiers, used by Alembic.π
+revision: str = '004'
+down_revision: Union[str, Sequence[str], None] = '003'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade():
+    """Add columns for whether the cluster is managed."""
+    with op.get_context().autocommit_block():
+        db_utils.add_column_to_table_alembic('clusters',
+                                             'is_managed',
+                                             sa.Integer(),
+                                             server_default='0')
+
+
+def downgrade():
+    """Remove columns for whether the cluster is managed."""
+    pass

sky/schemas/db/serve_state/001_initial_schema.py

@@ -0,0 +1,67 @@
+"""Initial schema for sky serve state database with backwards compatibility
+
+Revision ID: 001
+Revises:
+Create Date: 2024-01-01 12:00:00.000000
+
+"""
+# pylint: disable=invalid-name
+import json
+
+from alembic import op
+import sqlalchemy as sa
+
+from sky.serve import constants
+from sky.serve.serve_state import Base
+from sky.utils.db import db_utils
+
+# revision identifiers, used by Alembic.
+revision = '001'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    """Create initial schema and add all backwards compatibility columns"""
+    with op.get_context().autocommit_block():
+        # Create all tables with their current schema
+        db_utils.add_tables_to_db_sqlalchemy(Base.metadata, op.get_bind())
+
+        # Add backwards compatibility columns using helper function that matches
+        # original add_column_to_table_sqlalchemy behavior exactly
+        db_utils.add_column_to_table_alembic('services',
+                                             'requested_resources_str',
+                                             sa.Text())
+        db_utils.add_column_to_table_alembic(
+            'services',
+            'current_version',
+            sa.Integer(),
+            server_default=f'{constants.INITIAL_VERSION}')
+        db_utils.add_column_to_table_alembic('services',
+                                             'active_versions',
+                                             sa.Text(),
+                                             server_default=json.dumps([]))
+        db_utils.add_column_to_table_alembic('services',
+                                             'load_balancing_policy', sa.Text())
+        db_utils.add_column_to_table_alembic('services',
+                                             'tls_encrypted',
+                                             sa.Integer(),
+                                             server_default='0')
+        db_utils.add_column_to_table_alembic('services',
+                                             'pool',
+                                             sa.Integer(),
+                                             server_default='0')
+        db_utils.add_column_to_table_alembic(
+            'services',
+            'controller_pid',
+            sa.Integer(),
+            value_to_replace_existing_entries=-1)
+        db_utils.add_column_to_table_alembic('services', 'hash', sa.Text())
+        db_utils.add_column_to_table_alembic('services', 'entrypoint',
+                                             sa.Text())
+
+
+def downgrade():
+    """Drop all tables"""
+    Base.metadata.drop_all(bind=op.get_bind())

sky/schemas/db/spot_jobs/003_pool_hash.py

@@ -0,0 +1,34 @@
+"""Adding a hash column for pool.
+
+Revision ID: 003
+Revises: 002
+Create Date: 2025-07-18
+
+"""
+# pylint: disable=invalid-name
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+from sky.utils.db import db_utils
+
+# revision identifiers, used by Alembic.
+revision: str = '003'
+down_revision: Union[str, Sequence[str], None] = '002'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade():
+    """Add columns for pool hash."""
+    with op.get_context().autocommit_block():
+        db_utils.add_column_to_table_alembic('job_info',
+                                             'pool_hash',
+                                             sa.Text(),
+                                             server_default=None)
+
+
+def downgrade():
+    """Remove columns for pool hash."""
+    pass

sky/serve/client/impl.py

@@ -1,7 +1,7 @@
 """Implementation of SDK for SkyServe."""
 import json
 import typing
-from typing import List, Optional, Union
+from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
 
 import click
 
@@ -12,6 +12,8 @@ from sky.utils import admin_policy_utils
 from sky.utils import dag_utils
 
 if typing.TYPE_CHECKING:
+    import io
+
     import sky
     from sky.serve import serve_utils
 
@@ -23,7 +25,7 @@ def up(
     # Internal only:
     # pylint: disable=invalid-name
     _need_confirmation: bool = False
-) -> server_common.RequestId:
+) -> server_common.RequestId[Tuple[str, str]]:
     assert not pool, 'Command `up` is not supported for pool.'
     # Avoid circular import.
     from sky.client import sdk  # pylint: disable=import-outside-toplevel
@@ -67,7 +69,7 @@ def update(
     # Internal only:
     # pylint: disable=invalid-name
     _need_confirmation: bool = False
-) -> server_common.RequestId:
+) -> server_common.RequestId[None]:
     assert not pool, 'Command `update` is not supported for pool.'
     # Avoid circular import.
     from sky.client import sdk  # pylint: disable=import-outside-toplevel
@@ -110,7 +112,7 @@ def apply(
     # Internal only:
     # pylint: disable=invalid-name
     _need_confirmation: bool = False
-) -> server_common.RequestId:
+) -> server_common.RequestId[None]:
     assert pool, 'Command `apply` is only supported for pool.'
     # Avoid circular import.
     from sky.client import sdk  # pylint: disable=import-outside-toplevel
@@ -151,7 +153,7 @@ def down(
     all: bool = False,  # pylint: disable=redefined-builtin
     purge: bool = False,
     pool: bool = False,
-) -> server_common.RequestId:
+) -> server_common.RequestId[None]:
     if pool:
         body = payloads.JobsPoolDownBody(
             pool_names=service_names,
@@ -175,7 +177,7 @@ def down(
 def status(
     service_names: Optional[Union[str, List[str]]],
     pool: bool = False,
-) -> server_common.RequestId:
+) -> server_common.RequestId[List[Dict[str, Any]]]:
     if pool:
         body = payloads.JobsPoolStatusBody(pool_names=service_names)
     else:
@@ -186,3 +188,88 @@ def status(
         json=json.loads(body.model_dump_json()),
         timeout=(5, None))
     return server_common.get_request_id(response)
+
+
+def tail_logs(service_name: str,
+              target: Union[str, 'serve_utils.ServiceComponent'],
+              replica_id: Optional[int] = None,
+              follow: bool = True,
+              output_stream: Optional['io.TextIOBase'] = None,
+              tail: Optional[int] = None,
+              pool: bool = False) -> None:
+    # Avoid circular import.
+    from sky.client import sdk  # pylint: disable=import-outside-toplevel
+
+    if pool:
+        body = payloads.JobsPoolLogsBody(
+            pool_name=service_name,
+            target=target,
+            worker_id=replica_id,
+            follow=follow,
+            tail=tail,
+        )
+    else:
+        body = payloads.ServeLogsBody(
+            service_name=service_name,
+            target=target,
+            replica_id=replica_id,
+            follow=follow,
+            tail=tail,
+        )
+    response = server_common.make_authenticated_request(
+        'POST',
+        '/jobs/pool_logs' if pool else '/serve/logs',
+        json=json.loads(body.model_dump_json()),
+        timeout=(5, None),
+        stream=True)
+    request_id: server_common.RequestId[None] = server_common.get_request_id(
+        response)
+    return sdk.stream_response(request_id=request_id,
+                               response=response,
+                               output_stream=output_stream,
+                               resumable=True)
+
+
+def sync_down_logs(service_name: str,
+                   local_dir: str,
+                   *,
+                   targets: Optional[Union[
+                       str, 'serve_utils.ServiceComponent',
+                       Sequence[Union[str,
+                                      'serve_utils.ServiceComponent']]]] = None,
+                   replica_ids: Optional[List[int]] = None,
+                   tail: Optional[int] = None,
+                   pool: bool = False) -> None:
+    # Avoid circular import.
+    from sky.client import sdk  # pylint: disable=import-outside-toplevel
+
+    if pool:
+        body = payloads.JobsPoolDownloadLogsBody(
+            pool_name=service_name,
+            local_dir=local_dir,
+            targets=targets,
+            worker_ids=replica_ids,
+            tail=tail,
+        )
+    else:
+        body = payloads.ServeDownloadLogsBody(
+            service_name=service_name,
+            # No need to set here, since the server will override it
+            # to a directory on the API server.
+            local_dir=local_dir,
+            targets=targets,
+            replica_ids=replica_ids,
+            tail=tail,
+        )
+    response = server_common.make_authenticated_request(
+        'POST',
+        '/jobs/pool_sync-down-logs' if pool else '/serve/sync-down-logs',
+        json=json.loads(body.model_dump_json()),
+        timeout=(5, None))
+    request_id: server_common.RequestId[str] = server_common.get_request_id(
+        response)
+    remote_dir = sdk.stream_and_get(request_id)
+
+    # Download from API server paths to the client's local_dir
+    client_common.download_logs_from_api_server([remote_dir], remote_dir,
+                                                local_dir)

sky/serve/client/sdk.py

@@ -1,9 +1,8 @@
 """SDK for SkyServe."""
 import json
 import typing
-from typing import List, Optional, Union
+from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
 
-from sky.client import common as client_common
 from sky.serve.client import impl
 from sky.server import common as server_common
 from sky.server import rest
@@ -27,7 +26,7 @@ def up(
     # Internal only:
     # pylint: disable=invalid-name
     _need_confirmation: bool = False
-) -> server_common.RequestId:
+) -> server_common.RequestId[Tuple[str, str]]:
     """Spins up a service.
 
     Please refer to the sky.cli.serve_up for the document.
@@ -62,7 +61,7 @@ def update(
     # Internal only:
     # pylint: disable=invalid-name
     _need_confirmation: bool = False
-) -> server_common.RequestId:
+) -> server_common.RequestId[None]:
     """Updates an existing service.
 
     Please refer to the sky.cli.serve_update for the document.
@@ -95,7 +94,7 @@ def down(
     service_names: Optional[Union[str, List[str]]],
     all: bool = False,  # pylint: disable=redefined-builtin
     purge: bool = False
-) -> server_common.RequestId:
+) -> server_common.RequestId[None]:
     """Tears down a service.
 
     Please refer to the sky.cli.serve_down for the docs.
@@ -123,7 +122,7 @@ def down(
 @usage_lib.entrypoint
 @server_common.check_server_healthy_or_start
 def terminate_replica(service_name: str, replica_id: int,
-                      purge: bool) -> server_common.RequestId:
+                      purge: bool) -> server_common.RequestId[None]:
     """Tears down a specific replica for the given service.
 
     Args:
@@ -157,8 +156,8 @@ def terminate_replica(service_name: str, replica_id: int,
 @usage_lib.entrypoint
 @server_common.check_server_healthy_or_start
 def status(
-        service_names: Optional[Union[str,
-                                      List[str]]]) -> server_common.RequestId:
+    service_names: Optional[Union[str, List[str]]]
+) -> server_common.RequestId[List[Dict[str, Any]]]:
     """Gets service statuses.
 
     If service_names is given, return those services. Otherwise, return all
@@ -290,27 +289,13 @@ def tail_logs(service_name: str,
         sky.exceptions.ClusterNotUpError: the sky serve controller is not up.
         ValueError: arguments not valid, or failed to tail the logs.
     """
-    # Avoid circular import.
-    from sky.client import sdk  # pylint: disable=import-outside-toplevel
-
-    body = payloads.ServeLogsBody(
-        service_name=service_name,
-        target=target,
-        replica_id=replica_id,
-        follow=follow,
-        tail=tail,
-    )
-    response = server_common.make_authenticated_request(
-        'POST',
-        '/serve/logs',
-        json=json.loads(body.model_dump_json()),
-        timeout=(5, None),
-        stream=True)
-    request_id = server_common.get_request_id(response)
-    return sdk.stream_response(request_id=request_id,
-                               response=response,
-                               output_stream=output_stream,
-                               resumable=True)
+    return impl.tail_logs(service_name,
+                          target,
+                          replica_id,
+                          follow,
+                          output_stream,
+                          tail,
+                          pool=False)
 
 
 @usage_lib.entrypoint
@@ -320,8 +305,8 @@ def sync_down_logs(service_name: str,
                    *,
                    targets: Optional[Union[
                        str, 'serve_utils.ServiceComponent',
-                       List[Union[str,
-                                  'serve_utils.ServiceComponent']]]] = None,
+                       Sequence[Union[str,
+                                      'serve_utils.ServiceComponent']]]] = None,
                    replica_ids: Optional[List[int]] = None,
                    tail: Optional[int] = None) -> None:
     """Sync down logs from the service components to a local directory.
@@ -352,25 +337,9 @@ def sync_down_logs(service_name: str,
         sky.exceptions.ClusterNotUpError: If the controller is not up.
         ValueError: Arguments not valid.
     """
-    # Avoid circular import.
-    from sky.client import sdk  # pylint: disable=import-outside-toplevel
-
-    body = payloads.ServeDownloadLogsBody(
-        service_name=service_name,
-        # No need to set here, since the server will override it
-        # to a directory on the API server.
-        local_dir=local_dir,
-        targets=targets,
-        replica_ids=replica_ids,
-        tail=tail,
-    )
-    response = server_common.make_authenticated_request(
-        'POST',
-        '/serve/sync-down-logs',
-        json=json.loads(body.model_dump_json()),
-        timeout=(5, None))
-    remote_dir = sdk.stream_and_get(server_common.get_request_id(response))
-
-    # Download from API server paths to the client's local_dir
-    client_common.download_logs_from_api_server([remote_dir], remote_dir,
-                                                local_dir)
+    return impl.sync_down_logs(service_name,
+                               local_dir,
+                               targets=targets,
+                               replica_ids=replica_ids,
+                               tail=tail,
+                               pool=False)

sky/serve/constants.py

@@ -106,7 +106,8 @@ REPLICA_ID_ENV_VAR = 'SKYPILOT_SERVE_REPLICA_ID'
 # v2.0 - Added template-replica feature.
 # v3.0 - Added cluster pool.
 # v4.0 - Added pool argument to wait_service_registration.
-SERVE_VERSION = 4
+# v5.0 - Added pool argument to stream_serve_process_logs & stream_replica_logs.
+SERVE_VERSION = 5
 
 TERMINATE_REPLICA_VERSION_MISMATCH_ERROR = (
     'The version of service is outdated and does not support manually '

sky/serve/controller.py

@@ -27,11 +27,12 @@ from sky.utils import ux_utils
 logger = sky_logging.init_logger(__name__)
 
 
-class SuppressSuccessGetAccessLogsFilter(logging.Filter):
+class AutoscalerInfoFilter(logging.Filter):
 
     def filter(self, record: logging.LogRecord) -> bool:
         message = record.getMessage()
-        return not ('GET' in message and '200' in message)
+        return not ('GET' in message and '200' in message and
+                    '/autoscaler/info' in message)
 
 
 class SkyServeController:
@@ -61,6 +62,7 @@ class SkyServeController:
         uvicorn_access_logger = logging.getLogger('uvicorn.access')
         for handler in uvicorn_access_logger.handlers:
             handler.setFormatter(sky_logging.FORMATTER)
+            handler.addFilter(AutoscalerInfoFilter())
         yield
 
     def _run_autoscaler(self):