skypilot-nightly 1.0.0.dev20250804__py3-none-any.whl → 1.0.0.dev20250807__py3-none-any.whl

sky/provision/kubernetes/utils.py

@@ -2032,9 +2032,7 @@ class KubernetesInstanceType:
             accelerator_type = match.group('accelerator_type')
             if accelerator_count:
                 accelerator_count = int(accelerator_count)
-                # This is to revert the accelerator types with spaces back to
-                # the original format.
-                accelerator_type = str(accelerator_type).replace('_', ' ')
+                accelerator_type = str(accelerator_type)
             else:
                 accelerator_count = None
                 accelerator_type = None
@@ -2047,7 +2045,7 @@ class KubernetesInstanceType:
             accelerator_type = prev_match.group('accelerator_type')
             if accelerator_count:
                 accelerator_count = int(accelerator_count)
-                accelerator_type = str(accelerator_type).replace('_', ' ')
+                accelerator_type = str(accelerator_type)
             else:
                 accelerator_count = None
                 accelerator_type = None
@@ -2998,6 +2996,13 @@ def get_kubernetes_node_info(
                 # Get all the pods running on the node
                 if (pod.spec.node_name == node.metadata.name and
                         pod.status.phase in ['Running', 'Pending']):
+                    # Skip pods that should not count against GPU count
+                    if should_exclude_pod_from_gpu_allocation(pod):
+                        logger.debug(
+                            f'Excluding low priority pod '
+                            f'{pod.metadata.name} from GPU allocation '
+                            f'calculations on node {node.metadata.name}')
+                        continue
                     # Iterate over all the containers in the pod and sum the
                     # GPU requests
                     for container in pod.spec.containers:
@@ -3596,3 +3601,24 @@ def delete_k8s_resource_with_retry(delete_func: Callable, resource_type: str,
                 time.sleep(retry_delay)
             else:
                 raise
+
+
+def should_exclude_pod_from_gpu_allocation(pod) -> bool:
+    """Check if a pod should be excluded from GPU count calculations.
+
+    Some cloud providers run low priority test/verification pods that request
+    GPUs but should not count against real GPU availability since they are
+    designed to be evicted when higher priority workloads need resources.
+
+    Args:
+        pod: Kubernetes pod object
+
+    Returns:
+        bool: True if the pod should be excluded from GPU count calculations.
+    """
+    # CoreWeave HPC verification pods - identified by namespace
+    if (hasattr(pod.metadata, 'namespace') and
+            pod.metadata.namespace == 'cw-hpc-verification'):
+        return True
+
+    return False

sky/provision/nebius/instance.py

@@ -134,6 +134,7 @@ def run_instances(region: str, cluster_name_on_cloud: str,
                 image_family=config.node_config['ImageId'],
                 disk_size=config.node_config['DiskSize'],
                 user_data=config.node_config['UserData'],
+                use_spot=config.node_config['use_spot'],
                 associate_public_ip_address=(
                     not config.provider_config['use_internal_ips']),
                 filesystems=config.node_config.get('filesystems', []),

sky/provision/nebius/utils.py

@@ -168,6 +168,7 @@ def launch(cluster_name_on_cloud: str,
            user_data: str,
            associate_public_ip_address: bool,
            filesystems: List[Dict[str, Any]],
+           use_spot: bool = False,
            network_tier: Optional[resources_utils.NetworkTier] = None) -> str:
     # Each node must have a unique name to avoid conflicts between
     # multiple worker VMs. To ensure uniqueness,a UUID is appended
@@ -281,7 +282,14 @@ def launch(cluster_name_on_cloud: str,
                     public_ip_address=nebius.compute().PublicIPAddress()
                     if associate_public_ip_address else None,
                 )
-            ]))).wait()
+            ],
+            recovery_policy=nebius.compute().InstanceRecoveryPolicy.FAIL
+            if use_spot else None,
+            preemptible=nebius.compute().PreemptibleSpec(
+                priority=1,
+                on_preemption=nebius.compute(
+                ).PreemptibleSpec.PreemptionPolicy.STOP) if use_spot else None,
+        ))).wait()
     instance_id = ''
     retry_count = 0
     while retry_count < nebius.MAX_RETRIES_TO_INSTANCE_READY:

sky/schemas/db/global_user_state/002_add_workspace_to_cluster_history.py

@@ -0,0 +1,35 @@
+"""add workspace column to cluster_history table
+
+Revision ID: 002
+Revises: 001
+Create Date: 2025-08-06
+
+"""
+# pylint: disable=invalid-name
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+from sky.utils.db import db_utils
+
+# revision identifiers, used by Alembic.
+revision: str = '002'
+down_revision: Union[str, Sequence[str], None] = '001'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    with op.get_context().autocommit_block():
+        db_utils.add_column_to_table_alembic('cluster_history',
+                                             'workspace',
+                                             sa.Text(),
+                                             server_default=None)
+    pass
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    pass

sky/schemas/db/spot_jobs/003_pool_hash.py

@@ -0,0 +1,34 @@
+"""Adding a hash column for pool.
+
+Revision ID: 003
+Revises: 002
+Create Date: 2025-07-18
+
+"""
+# pylint: disable=invalid-name
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+from sky.utils.db import db_utils
+
+# revision identifiers, used by Alembic.
+revision: str = '003'
+down_revision: Union[str, Sequence[str], None] = '002'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade():
+    """Add columns for pool hash."""
+    with op.get_context().autocommit_block():
+        db_utils.add_column_to_table_alembic('job_info',
+                                             'pool_hash',
+                                             sa.Text(),
+                                             server_default=None)
+
+
+def downgrade():
+    """Remove columns for pool hash."""
+    pass

sky/serve/client/impl.py

@@ -1,7 +1,7 @@
 """Implementation of SDK for SkyServe."""
 import json
 import typing
-from typing import List, Optional, Union
+from typing import List, Optional, Sequence, Union
 
 import click
 
@@ -12,6 +12,8 @@ from sky.utils import admin_policy_utils
 from sky.utils import dag_utils
 
 if typing.TYPE_CHECKING:
+    import io
+
     import sky
     from sky.serve import serve_utils
 
@@ -186,3 +188,85 @@ def status(
         json=json.loads(body.model_dump_json()),
         timeout=(5, None))
     return server_common.get_request_id(response)
+
+
+def tail_logs(service_name: str,
+              target: Union[str, 'serve_utils.ServiceComponent'],
+              replica_id: Optional[int] = None,
+              follow: bool = True,
+              output_stream: Optional['io.TextIOBase'] = None,
+              tail: Optional[int] = None,
+              pool: bool = False) -> None:
+    # Avoid circular import.
+    from sky.client import sdk  # pylint: disable=import-outside-toplevel
+
+    if pool:
+        body = payloads.JobsPoolLogsBody(
+            pool_name=service_name,
+            target=target,
+            worker_id=replica_id,
+            follow=follow,
+            tail=tail,
+        )
+    else:
+        body = payloads.ServeLogsBody(
+            service_name=service_name,
+            target=target,
+            replica_id=replica_id,
+            follow=follow,
+            tail=tail,
+        )
+    response = server_common.make_authenticated_request(
+        'POST',
+        '/jobs/pool_logs' if pool else '/serve/logs',
+        json=json.loads(body.model_dump_json()),
+        timeout=(5, None),
+        stream=True)
+    request_id = server_common.get_request_id(response)
+    return sdk.stream_response(request_id=request_id,
+                               response=response,
+                               output_stream=output_stream,
+                               resumable=True)
+
+
+def sync_down_logs(service_name: str,
+                   local_dir: str,
+                   *,
+                   targets: Optional[Union[
+                       str, 'serve_utils.ServiceComponent',
+                       Sequence[Union[str,
+                                      'serve_utils.ServiceComponent']]]] = None,
+                   replica_ids: Optional[List[int]] = None,
+                   tail: Optional[int] = None,
+                   pool: bool = False) -> None:
+    # Avoid circular import.
+    from sky.client import sdk  # pylint: disable=import-outside-toplevel
+
+    if pool:
+        body = payloads.JobsPoolDownloadLogsBody(
+            pool_name=service_name,
+            local_dir=local_dir,
+            targets=targets,
+            worker_ids=replica_ids,
+            tail=tail,
+        )
+    else:
+        body = payloads.ServeDownloadLogsBody(
+            service_name=service_name,
+            # No need to set here, since the server will override it
+            # to a directory on the API server.
+            local_dir=local_dir,
+            targets=targets,
+            replica_ids=replica_ids,
+            tail=tail,
+        )
+    response = server_common.make_authenticated_request(
+        'POST',
+        '/jobs/pool_sync-down-logs' if pool else '/serve/sync-down-logs',
+        json=json.loads(body.model_dump_json()),
+        timeout=(5, None))
+    remote_dir = sdk.stream_and_get(server_common.get_request_id(response))
+
+    # Download from API server paths to the client's local_dir
+    client_common.download_logs_from_api_server([remote_dir], remote_dir,
+                                                local_dir)

sky/serve/client/sdk.py

@@ -1,9 +1,8 @@
 """SDK for SkyServe."""
 import json
 import typing
-from typing import List, Optional, Union
+from typing import List, Optional, Sequence, Union
 
-from sky.client import common as client_common
 from sky.serve.client import impl
 from sky.server import common as server_common
 from sky.server import rest
@@ -290,27 +289,13 @@ def tail_logs(service_name: str,
         sky.exceptions.ClusterNotUpError: the sky serve controller is not up.
         ValueError: arguments not valid, or failed to tail the logs.
     """
-    # Avoid circular import.
-    from sky.client import sdk  # pylint: disable=import-outside-toplevel
-
-    body = payloads.ServeLogsBody(
-        service_name=service_name,
-        target=target,
-        replica_id=replica_id,
-        follow=follow,
-        tail=tail,
-    )
-    response = server_common.make_authenticated_request(
-        'POST',
-        '/serve/logs',
-        json=json.loads(body.model_dump_json()),
-        timeout=(5, None),
-        stream=True)
-    request_id = server_common.get_request_id(response)
-    return sdk.stream_response(request_id=request_id,
-                               response=response,
-                               output_stream=output_stream,
-                               resumable=True)
+    return impl.tail_logs(service_name,
+                          target,
+                          replica_id,
+                          follow,
+                          output_stream,
+                          tail,
+                          pool=False)
 
 
 @usage_lib.entrypoint
@@ -320,8 +305,8 @@ def sync_down_logs(service_name: str,
                    *,
                    targets: Optional[Union[
                        str, 'serve_utils.ServiceComponent',
-                       List[Union[str,
-                                  'serve_utils.ServiceComponent']]]] = None,
+                       Sequence[Union[str,
+                                      'serve_utils.ServiceComponent']]]] = None,
                    replica_ids: Optional[List[int]] = None,
                    tail: Optional[int] = None) -> None:
     """Sync down logs from the service components to a local directory.
@@ -352,25 +337,9 @@ def sync_down_logs(service_name: str,
         sky.exceptions.ClusterNotUpError: If the controller is not up.
         ValueError: Arguments not valid.
     """
-    # Avoid circular import.
-    from sky.client import sdk  # pylint: disable=import-outside-toplevel
-
-    body = payloads.ServeDownloadLogsBody(
-        service_name=service_name,
-        # No need to set here, since the server will override it
-        # to a directory on the API server.
-        local_dir=local_dir,
-        targets=targets,
-        replica_ids=replica_ids,
-        tail=tail,
-    )
-    response = server_common.make_authenticated_request(
-        'POST',
-        '/serve/sync-down-logs',
-        json=json.loads(body.model_dump_json()),
-        timeout=(5, None))
-    remote_dir = sdk.stream_and_get(server_common.get_request_id(response))
-
-    # Download from API server paths to the client's local_dir
-    client_common.download_logs_from_api_server([remote_dir], remote_dir,
-                                                local_dir)
+    return impl.sync_down_logs(service_name,
+                               local_dir,
+                               targets=targets,
+                               replica_ids=replica_ids,
+                               tail=tail,
+                               pool=False)

sky/serve/client/sdk_async.py

@@ -0,0 +1,130 @@
+"""Async SDK for SkyServe."""
+import typing
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+from sky.client import sdk_async
+from sky.serve.client import sdk
+from sky.usage import usage_lib
+from sky.utils import context_utils
+
+if typing.TYPE_CHECKING:
+    import io
+
+    import sky
+    from sky.serve import serve_utils
+
+
+@usage_lib.entrypoint
+async def up(
+    task: Union['sky.Task', 'sky.Dag'],
+    service_name: str,
+    # Internal only:
+    # pylint: disable=invalid-name
+    _need_confirmation: bool = False,
+    stream_logs: Optional[
+        sdk_async.StreamConfig] = sdk_async.DEFAULT_STREAM_CONFIG
+) -> Tuple[str, str]:
+    """Async version of up() that spins up a service."""
+    request_id = await context_utils.to_thread(sdk.up, task, service_name,
+                                               _need_confirmation)
+    if stream_logs is not None:
+        return await sdk_async._stream_and_get(request_id, stream_logs)  # pylint: disable=protected-access
+    else:
+        return await sdk_async.get(request_id)
+
+
+@usage_lib.entrypoint
+async def update(
+    task: Union['sky.Task', 'sky.Dag'],
+    service_name: str,
+    mode: 'serve_utils.UpdateMode',
+    # Internal only:
+    # pylint: disable=invalid-name
+    _need_confirmation: bool = False,
+    stream_logs: Optional[
+        sdk_async.StreamConfig] = sdk_async.DEFAULT_STREAM_CONFIG
+) -> None:
+    """Async version of update() that updates an existing service."""
+    request_id = await context_utils.to_thread(sdk.update, task, service_name,
+                                               mode, _need_confirmation)
+    if stream_logs is not None:
+        return await sdk_async._stream_and_get(request_id, stream_logs)  # pylint: disable=protected-access
+    else:
+        return await sdk_async.get(request_id)
+
+
+@usage_lib.entrypoint
+async def down(
+    service_names: Optional[Union[str, List[str]]],
+    all: bool = False,  # pylint: disable=redefined-builtin
+    purge: bool = False,
+    stream_logs: Optional[
+        sdk_async.StreamConfig] = sdk_async.DEFAULT_STREAM_CONFIG
+) -> None:
+    """Async version of down() that tears down a service."""
+    request_id = await context_utils.to_thread(sdk.down, service_names, all,
+                                               purge)
+    if stream_logs is not None:
+        return await sdk_async._stream_and_get(request_id, stream_logs)  # pylint: disable=protected-access
+    else:
+        return await sdk_async.get(request_id)
+
+
+@usage_lib.entrypoint
+async def terminate_replica(
+    service_name: str,
+    replica_id: int,
+    purge: bool,
+    stream_logs: Optional[
+        sdk_async.StreamConfig] = sdk_async.DEFAULT_STREAM_CONFIG
+) -> None:
+    """Async version of terminate_replica() that tears down a specific
+    replica."""
+    request_id = await context_utils.to_thread(sdk.terminate_replica,
+                                               service_name, replica_id, purge)
+    if stream_logs is not None:
+        return await sdk_async._stream_and_get(request_id, stream_logs)  # pylint: disable=protected-access
+    else:
+        return await sdk_async.get(request_id)
+
+
+@usage_lib.entrypoint
+async def status(
+    service_names: Optional[Union[str, List[str]]],
+    stream_logs: Optional[
+        sdk_async.StreamConfig] = sdk_async.DEFAULT_STREAM_CONFIG
+) -> List[Dict[str, Any]]:
+    """Async version of status() that sdk_async.gets service statuses."""
+    request_id = await context_utils.to_thread(sdk.status, service_names)
+    if stream_logs is not None:
+        return await sdk_async._stream_and_get(request_id, stream_logs)  # pylint: disable=protected-access
+    else:
+        return await sdk_async.get(request_id)
+
+
+@usage_lib.entrypoint
+async def tail_logs(service_name: str,
+                    target: Union[str, 'serve_utils.ServiceComponent'],
+                    replica_id: Optional[int] = None,
+                    follow: bool = True,
+                    output_stream: Optional['io.TextIOBase'] = None) -> None:
+    """Async version of tail_logs() that tails logs for a service."""
+    return await context_utils.to_thread(sdk.tail_logs, service_name, target,
+                                         replica_id, follow, output_stream)
+
+
+@usage_lib.entrypoint
+async def sync_down_logs(service_name: str,
+                         local_dir: str,
+                         *,
+                         targets: Optional[Union[
+                             str, 'serve_utils.ServiceComponent', List[Union[
+                                 str, 'serve_utils.ServiceComponent']]]] = None,
+                         replica_ids: Optional[List[int]] = None) -> None:
+    """Async version of sync_down_logs() that syncs down logs from service
+      components."""
+    return await context_utils.to_thread(sdk.sync_down_logs,
+                                         service_name,
+                                         local_dir,
+                                         targets=targets,
+                                         replica_ids=replica_ids)

sky/serve/constants.py

@@ -105,7 +105,9 @@ REPLICA_ID_ENV_VAR = 'SKYPILOT_SERVE_REPLICA_ID'
 # v1.0 - Introduce rolling update.
 # v2.0 - Added template-replica feature.
 # v3.0 - Added cluster pool.
-SERVE_VERSION = 3
+# v4.0 - Added pool argument to wait_service_registration.
+# v5.0 - Added pool argument to stream_serve_process_logs & stream_replica_logs.
+SERVE_VERSION = 5
 
 TERMINATE_REPLICA_VERSION_MISMATCH_ERROR = (
     'The version of service is outdated and does not support manually '

sky/serve/controller.py

@@ -4,6 +4,7 @@ Responsible for autoscaling and replica management.
 """
 import contextlib
 import logging
+import os
 import threading
 import time
 import traceback
@@ -26,11 +27,12 @@ from sky.utils import ux_utils
 logger = sky_logging.init_logger(__name__)
 
 
-class SuppressSuccessGetAccessLogsFilter(logging.Filter):
+class AutoscalerInfoFilter(logging.Filter):
 
     def filter(self, record: logging.LogRecord) -> bool:
         message = record.getMessage()
-        return not ('GET' in message and '200' in message)
+        return not ('GET' in message and '200' in message and
+                    '/autoscaler/info' in message)
 
 
 class SkyServeController:
@@ -60,6 +62,7 @@ class SkyServeController:
         uvicorn_access_logger = logging.getLogger('uvicorn.access')
         for handler in uvicorn_access_logger.handlers:
             handler.setFormatter(sky_logging.FORMATTER)
+            handler.addFilter(AutoscalerInfoFilter())
         yield
 
     def _run_autoscaler(self):
@@ -242,7 +245,7 @@ class SkyServeController:
         threading.Thread(target=self._run_autoscaler).start()
 
         logger.info('SkyServe Controller started on '
-                    f'http://{self._host}:{self._port}')
+                    f'http://{self._host}:{self._port}. PID: {os.getpid()}')
 
         uvicorn.run(self._app, host=self._host, port=self._port)
 

sky/serve/load_balancer.py

@@ -1,6 +1,7 @@
 """LoadBalancer: Distribute any incoming request to all ready replicas."""
 import asyncio
 import logging
+import os
 import threading
 import traceback
 from typing import Dict, List, Optional, Union
@@ -254,7 +255,8 @@ class SkyServeLoadBalancer:
         protocol = 'https' if self._tls_credential is not None else 'http'
 
         logger.info('SkyServe Load Balancer started on '
-                    f'{protocol}://0.0.0.0:{self._load_balancer_port}')
+                    f'{protocol}://0.0.0.0:{self._load_balancer_port}. '
+                    f'PID: {os.getpid()}')
 
         uvicorn.run(self._app,
                     host='0.0.0.0',