skypilot-nightly 1.0.0.dev20250927__py3-none-any.whl → 1.0.0.dev20251002__py3-none-any.whl

sky/jobs/server/core.py

@@ -28,6 +28,7 @@ from sky.jobs import constants as managed_job_constants
 from sky.jobs import state as managed_job_state
 from sky.jobs import utils as managed_job_utils
 from sky.provision import common as provision_common
+from sky.schemas.api import responses
 from sky.serve import serve_state
 from sky.serve import serve_utils
 from sky.serve.server import impl
@@ -296,8 +297,7 @@ def launch(
                 # TODO: do something with returned status?
                 _, _ = backend_utils.refresh_cluster_status_handle(
                     cluster_name=cluster_name,
-                    force_refresh_statuses=set(status_lib.ClusterStatus),
-                    acquire_per_cluster_status_lock=False)
+                    force_refresh_statuses=set(status_lib.ClusterStatus))
             except (exceptions.ClusterOwnerIdentityMismatchError,
                     exceptions.CloudUserIdentityError,
                     exceptions.ClusterStatusFetchingError) as e:
@@ -644,6 +644,28 @@ def queue(refresh: bool,
 
 
 @usage_lib.entrypoint
+def queue_v2_api(
+    refresh: bool,
+    skip_finished: bool = False,
+    all_users: bool = False,
+    job_ids: Optional[List[int]] = None,
+    user_match: Optional[str] = None,
+    workspace_match: Optional[str] = None,
+    name_match: Optional[str] = None,
+    pool_match: Optional[str] = None,
+    page: Optional[int] = None,
+    limit: Optional[int] = None,
+    statuses: Optional[List[str]] = None,
+) -> Tuple[List[responses.ManagedJobRecord], int, Dict[str, int], int]:
+    """Gets statuses of managed jobs and parse the
+    jobs to responses.ManagedJobRecord."""
+    jobs, total, status_counts, total_no_filter = queue_v2(
+        refresh, skip_finished, all_users, job_ids, user_match, workspace_match,
+        name_match, pool_match, page, limit, statuses)
+    return [responses.ManagedJobRecord(**job) for job in jobs
+           ], total, status_counts, total_no_filter
+
+
 def queue_v2(
     refresh: bool,
     skip_finished: bool = False,

sky/jobs/server/server.py

@@ -68,7 +68,7 @@ async def queue_v2(request: fastapi.Request,
         request_id=request.state.request_id,
         request_name='jobs.queue_v2',
         request_body=jobs_queue_body_v2,
-        func=core.queue_v2,
+        func=core.queue_v2_api,
         schedule_type=(api_requests.ScheduleType.LONG
                        if jobs_queue_body_v2.refresh else
                        api_requests.ScheduleType.SHORT),

sky/jobs/utils.py

@@ -33,6 +33,7 @@ from sky.backends import cloud_vm_ray_backend
 from sky.jobs import constants as managed_job_constants
 from sky.jobs import scheduler
 from sky.jobs import state as managed_job_state
+from sky.schemas.api import responses
 from sky.skylet import constants
 from sky.skylet import job_lib
 from sky.skylet import log_lib
@@ -1517,7 +1518,7 @@ def load_managed_job_queue(
 
 
 def _get_job_status_from_tasks(
-    job_tasks: List[Dict[str, Any]]
+    job_tasks: Union[List[responses.ManagedJobRecord], List[Dict[str, Any]]]
 ) -> Tuple[managed_job_state.ManagedJobStatus, int]:
     """Get the current task status and the current task id for a job."""
     managed_task_status = managed_job_state.ManagedJobStatus.SUCCEEDED

sky/provision/kubernetes/instance.py

@@ -847,7 +847,7 @@ def _create_namespaced_pod_with_retries(namespace: str, pod_spec: dict,
 def _wait_for_deployment_pod(context,
                              namespace,
                              deployment,
-                             timeout=60) -> List:
+                             timeout=300) -> List:
     label_selector = ','.join([
         f'{key}={value}'
         for key, value in deployment.spec.selector.match_labels.items()

sky/provision/kubernetes/utils.py

@@ -1,4 +1,5 @@
 """Kubernetes utilities for SkyPilot."""
+import collections
 import copy
 import dataclasses
 import datetime
@@ -3117,14 +3118,6 @@ def get_kubernetes_node_info(
             information.
     """
     nodes = get_kubernetes_nodes(context=context)
-    # Get the pods to get the real-time resource usage
-    try:
-        pods = get_all_pods_in_kubernetes_cluster(context=context)
-    except kubernetes.api_exception() as e:
-        if e.status == 403:
-            pods = None
-        else:
-            raise
 
     lf, _ = detect_gpu_label_formatter(context)
     if not lf:
@@ -3132,6 +3125,46 @@ def get_kubernetes_node_info(
     else:
         label_keys = lf.get_label_keys()
 
+    # Check if all nodes have no accelerators to avoid fetching pods
+    any_node_has_accelerators = False
+    for node in nodes:
+        accelerator_count = get_node_accelerator_count(context,
+                                                       node.status.allocatable)
+        if accelerator_count > 0:
+            any_node_has_accelerators = True
+            break
+
+    # Get the pods to get the real-time resource usage
+    pods = None
+    allocated_qty_by_node: Dict[str, int] = collections.defaultdict(int)
+    if any_node_has_accelerators:
+        try:
+            pods = get_all_pods_in_kubernetes_cluster(context=context)
+            # Pre-compute allocated accelerator count per node
+            for pod in pods:
+                if pod.status.phase in ['Running', 'Pending']:
+                    # Skip pods that should not count against GPU count
+                    if should_exclude_pod_from_gpu_allocation(pod):
+                        logger.debug(f'Excluding low priority pod '
+                                     f'{pod.metadata.name} from GPU allocation '
+                                     f'calculations')
+                        continue
+                    # Iterate over all the containers in the pod and sum the
+                    # GPU requests
+                    pod_allocated_qty = 0
+                    for container in pod.spec.containers:
+                        if container.resources.requests:
+                            pod_allocated_qty += get_node_accelerator_count(
+                                context, container.resources.requests)
+                    if pod_allocated_qty > 0:
+                        allocated_qty_by_node[
+                            pod.spec.node_name] += pod_allocated_qty
+        except kubernetes.api_exception() as e:
+            if e.status == 403:
+                pass
+            else:
+                raise
+
     node_info_dict: Dict[str, models.KubernetesNodeInfo] = {}
     has_multi_host_tpu = False
 
@@ -3161,32 +3194,21 @@ def get_kubernetes_node_info(
                         node_ip = address.address
                         break
 
-        allocated_qty = 0
         accelerator_count = get_node_accelerator_count(context,
                                                        node.status.allocatable)
+        if accelerator_count == 0:
+            node_info_dict[node.metadata.name] = models.KubernetesNodeInfo(
+                name=node.metadata.name,
+                accelerator_type=accelerator_name,
+                total={'accelerator_count': 0},
+                free={'accelerators_available': 0},
+                ip_address=node_ip)
+            continue
 
         if pods is None:
             accelerators_available = -1
-
         else:
-            for pod in pods:
-                # Get all the pods running on the node
-                if (pod.spec.node_name == node.metadata.name and
-                        pod.status.phase in ['Running', 'Pending']):
-                    # Skip pods that should not count against GPU count
-                    if should_exclude_pod_from_gpu_allocation(pod):
-                        logger.debug(
-                            f'Excluding low priority pod '
-                            f'{pod.metadata.name} from GPU allocation '
-                            f'calculations on node {node.metadata.name}')
-                        continue
-                    # Iterate over all the containers in the pod and sum the
-                    # GPU requests
-                    for container in pod.spec.containers:
-                        if container.resources.requests:
-                            allocated_qty += get_node_accelerator_count(
-                                context, container.resources.requests)
-
+            allocated_qty = allocated_qty_by_node[node.metadata.name]
             accelerators_available = accelerator_count - allocated_qty
 
         # Exclude multi-host TPUs from being processed.

sky/schemas/api/responses.py

@@ -5,7 +5,9 @@ from typing import Any, Dict, List, Optional
 
 import pydantic
 
+from sky import data
 from sky import models
+from sky.jobs import state as job_state
 from sky.server import common
 from sky.skylet import job_lib
 from sky.utils import status_lib
@@ -143,3 +145,77 @@ class UploadStatus(enum.Enum):
     """Status of the upload."""
     UPLOADING = 'uploading'
     COMPLETED = 'completed'
+
+
+class StorageRecord(ResponseBaseModel):
+    """Response for the storage list endpoint."""
+    name: str
+    launched_at: int
+    store: List[data.StoreType]
+    last_use: str
+    status: status_lib.StorageStatus
+
+
+# TODO (syang) figure out which fields are always present
+# and therefore can be non-optional.
+class ManagedJobRecord(ResponseBaseModel):
+    """A single managed job record."""
+    job_id: Optional[int] = None
+    task_id: Optional[int] = None
+    job_name: Optional[str] = None
+    task_name: Optional[str] = None
+    job_duration: Optional[float] = None
+    workspace: Optional[str] = None
+    status: Optional[job_state.ManagedJobStatus] = None
+    schedule_state: Optional[str] = None
+    resources: Optional[str] = None
+    cluster_resources: Optional[str] = None
+    cluster_resources_full: Optional[str] = None
+    cloud: Optional[str] = None
+    region: Optional[str] = None
+    zone: Optional[str] = None
+    infra: Optional[str] = None
+    recovery_count: Optional[int] = None
+    details: Optional[str] = None
+    failure_reason: Optional[str] = None
+    user_name: Optional[str] = None
+    user_hash: Optional[str] = None
+    submitted_at: Optional[float] = None
+    start_at: Optional[float] = None
+    end_at: Optional[float] = None
+    user_yaml: Optional[str] = None
+    entrypoint: Optional[str] = None
+    metadata: Optional[Dict[str, Any]] = None
+    controller_pid: Optional[int] = None
+    dag_yaml_path: Optional[str] = None
+    env_file_path: Optional[str] = None
+    last_recovered_at: Optional[float] = None
+    run_timestamp: Optional[str] = None
+    priority: Optional[int] = None
+    original_user_yaml_path: Optional[str] = None
+    pool: Optional[str] = None
+    pool_hash: Optional[str] = None
+    current_cluster_name: Optional[str] = None
+    job_id_on_pool_cluster: Optional[int] = None
+    accelerators: Optional[Dict[str, int]] = None
+
+
+class VolumeRecord(ResponseBaseModel):
+    """A single volume record."""
+    name: str
+    type: str
+    launched_at: int
+    cloud: str
+    region: str
+    zone: Optional[str] = None
+    size: str
+    config: Dict[str, Any]
+    name_on_cloud: str
+    user_hash: str
+    user_name: str
+    workspace: str
+    last_attached_at: Optional[int] = None
+    last_use: Optional[str] = None
+    status: Optional[str] = None
+    usedby_pods: List[str]
+    usedby_clusters: List[str]

sky/server/common.py

@@ -780,6 +780,7 @@ def check_server_healthy_or_start_fn(deploy: bool = False,
                 os.path.expanduser(constants.API_SERVER_CREATION_LOCK_PATH)):
             # Check again if server is already running. Other processes may
             # have started the server while we were waiting for the lock.
+            get_api_server_status.cache_clear()  # type: ignore[attr-defined]
             api_server_info = get_api_server_status(endpoint)
             if api_server_info.status == ApiServerStatus.UNHEALTHY:
                 _start_api_server(deploy, host, foreground, metrics,
@@ -841,7 +842,7 @@ def process_mounts_in_task_on_api_server(task: str, env_vars: Dict[str, str],
     for task_config in task_configs:
         if task_config is None:
             continue
-        file_mounts_mapping = task_config.get('file_mounts_mapping', {})
+        file_mounts_mapping = task_config.pop('file_mounts_mapping', {})
         if not file_mounts_mapping:
             # We did not mount any files to new paths on the remote server
             # so no need to resolve filepaths.

sky/server/requests/serializers/decoders.py

@@ -72,7 +72,7 @@ def decode_status_kubernetes(
                         List[Dict[str, Any]], Optional[str]]
 ) -> Tuple[List[kubernetes_utils.KubernetesSkyPilotClusterInfoPayload],
            List[kubernetes_utils.KubernetesSkyPilotClusterInfoPayload],
-           List[Dict[str, Any]], Optional[str]]:
+           List[responses.ManagedJobRecord], Optional[str]]:
     (encoded_all_clusters, encoded_unmanaged_clusters, all_jobs,
      context) = return_value
     all_clusters = []
@@ -85,6 +85,7 @@ def decode_status_kubernetes(
         cluster['status'] = status_lib.ClusterStatus(cluster['status'])
         unmanaged_clusters.append(
             kubernetes_utils.KubernetesSkyPilotClusterInfoPayload(**cluster))
+    all_jobs = [responses.ManagedJobRecord(**job) for job in all_jobs]
     return all_clusters, unmanaged_clusters, all_jobs, context
 
 
@@ -115,7 +116,7 @@ def decode_jobs_queue(return_value: List[dict],) -> List[Dict[str, Any]]:
 
 
 @register_decoders('jobs.queue_v2')
-def decode_jobs_queue_v2(return_value) -> List[Dict[str, Any]]:
+def decode_jobs_queue_v2(return_value) -> List[responses.ManagedJobRecord]:
     """Decode jobs queue response.
 
     Supports legacy list, or a dict {jobs, total}.
@@ -129,6 +130,7 @@ def decode_jobs_queue_v2(return_value) -> List[Dict[str, Any]]:
         jobs = return_value
     for job in jobs:
         job['status'] = managed_jobs.ManagedJobStatus(job['status'])
+    jobs = [responses.ManagedJobRecord(**job) for job in jobs]
     return jobs
 
 
@@ -181,14 +183,24 @@ def decode_list_accelerators(
 
 @register_decoders('storage_ls')
 def decode_storage_ls(
-        return_value: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        return_value: List[Dict[str, Any]]) -> List[responses.StorageRecord]:
     for storage_info in return_value:
         storage_info['status'] = status_lib.StorageStatus(
             storage_info['status'])
         storage_info['store'] = [
             storage.StoreType(store) for store in storage_info['store']
         ]
-    return return_value
+    return [
+        responses.StorageRecord(**storage_info) for storage_info in return_value
+    ]
+
+
+@register_decoders('volume_list')
+def decode_volume_list(
+        return_value: List[Dict[str, Any]]) -> List[responses.VolumeRecord]:
+    return [
+        responses.VolumeRecord(**volume_info) for volume_info in return_value
+    ]
 
 
 @register_decoders('job_status')

sky/server/requests/serializers/encoders.py

@@ -107,7 +107,7 @@ def encode_status_kubernetes(
     return_value: Tuple[
         List['kubernetes_utils.KubernetesSkyPilotClusterInfoPayload'],
         List['kubernetes_utils.KubernetesSkyPilotClusterInfoPayload'],
-        List[Dict[str, Any]], Optional[str]]
+        List[responses.ManagedJobRecord], Optional[str]]
 ) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], List[Dict[str, Any]],
            Optional[str]]:
     all_clusters, unmanaged_clusters, all_jobs, context = return_value
@@ -121,6 +121,7 @@ def encode_status_kubernetes(
         encoded_cluster = dataclasses.asdict(cluster)
         encoded_cluster['status'] = encoded_cluster['status'].value
         encoded_unmanaged_clusters.append(encoded_cluster)
+    all_jobs = [job.model_dump() for job in all_jobs]
     return encoded_all_clusters, encoded_unmanaged_clusters, all_jobs, context
 
 
@@ -150,9 +151,9 @@ def encode_jobs_queue_v2(
     for job in jobs:
         job['status'] = job['status'].value
     if total is None:
-        return jobs
+        return [job.model_dump() for job in jobs]
     return {
-        'jobs': jobs,
+        'jobs': [job.model_dump() for job in jobs],
         'total': total,
         'total_no_filter': total_no_filter,
         'status_counts': status_counts
@@ -203,11 +204,17 @@ def encode_enabled_clouds(clouds: List['clouds.Cloud']) -> List[str]:
 
 @register_encoder('storage_ls')
 def encode_storage_ls(
-        return_value: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        return_value: List[responses.StorageRecord]) -> List[Dict[str, Any]]:
     for storage_info in return_value:
         storage_info['status'] = storage_info['status'].value
         storage_info['store'] = [store.value for store in storage_info['store']]
-    return return_value
+    return [storage_info.model_dump() for storage_info in return_value]
+
+
+@register_encoder('volume_list')
+def encode_volume_list(
+        return_value: List[responses.VolumeRecord]) -> List[Dict[str, Any]]:
+    return [volume_info.model_dump() for volume_info in return_value]
 
 
 @register_encoder('job_status')

sky/task.py

@@ -649,6 +649,10 @@ class Task:
             config['workdir'] = _fill_in_env_vars(config['workdir'],
                                                   env_and_secrets)
 
+        if config.get('volumes') is not None:
+            config['volumes'] = _fill_in_env_vars(config['volumes'],
+                                                  env_and_secrets)
+
         task = Task(
             config.pop('name', None),
             run=config.pop('run', None),

sky/utils/cluster_utils.py

@@ -193,11 +193,29 @@ class SSHConfigHelper(object):
         proxy_command = auth_config.get('ssh_proxy_command', None)
 
         docker_proxy_command_generator = None
+        proxy_command_for_nodes = proxy_command
         if docker_user is not None:
-            docker_proxy_command_generator = lambda ip, port: ' '.join(
-                ['ssh'] + command_runner.ssh_options_list(
-                    key_path, ssh_control_name=None, port=port) +
-                ['-W', '%h:%p', f'{auth_config["ssh_user"]}@{ip}'])
+
+            def _docker_proxy_cmd(ip: str, port: int) -> str:
+                inner_proxy = proxy_command
+                inner_port = port or 22
+                if inner_proxy is not None:
+                    inner_proxy = inner_proxy.replace('%h', ip)
+                    inner_proxy = inner_proxy.replace('%p', str(inner_port))
+                return ' '.join(['ssh'] + command_runner.ssh_options_list(
+                    key_path,
+                    ssh_control_name=None,
+                    ssh_proxy_command=inner_proxy,
+                    port=inner_port,
+                    # ProxyCommand (ssh -W) is a forwarding tunnel, not an
+                    # interactive session. ControlMaster would cache these
+                    # processes, causing them to hang and block subsequent
+                    # connections. Each ProxyCommand should be ephemeral.
+                    disable_control_master=True
+                ) + ['-W', '%h:%p', f'{auth_config["ssh_user"]}@{ip}'])
+
+            docker_proxy_command_generator = _docker_proxy_cmd
+            proxy_command_for_nodes = None
 
         codegen = ''
         # Add the nodes to the codegen
@@ -212,7 +230,7 @@ class SSHConfigHelper(object):
             # TODO(romilb): Update port number when k8s supports multinode
             codegen += cls._get_generated_config(
                 sky_autogen_comment, node_name, ip, username,
-                key_path_for_config, proxy_command, port,
+                key_path_for_config, proxy_command_for_nodes, port,
                 docker_proxy_command) + '\n'
 
         cluster_config_path = os.path.expanduser(

sky/utils/command_runner.py

@@ -652,15 +652,31 @@ class SSHCommandRunner(CommandRunner):
         if docker_user is not None:
             assert port is None or port == 22, (
                 f'port must be None or 22 for docker_user, got {port}.')
-            # Already checked in resources
-            assert ssh_proxy_command is None, (
-                'ssh_proxy_command is not supported when using docker.')
+            # When connecting via docker, the outer SSH hop points to the
+            # container's sshd (localhost). Preserve the user proxy for the
+            # inner hop that reaches the host VM, and clear the outer proxy to
+            # avoid forwarding localhost through the jump host.
+            inner_proxy_command = ssh_proxy_command
+            inner_proxy_port = port or 22
+            self._ssh_proxy_command = None
             self.ip = 'localhost'
             self.ssh_user = docker_user
             self.port = constants.DEFAULT_DOCKER_PORT
+            if inner_proxy_command is not None:
+                # Replace %h/%p placeholders with actual host values, since the
+                # final destination from the perspective of the user proxy is
+                # the host VM (ip, inner_proxy_port).
+                inner_proxy_command = inner_proxy_command.replace('%h', ip)
+                inner_proxy_command = inner_proxy_command.replace(
+                    '%p', str(inner_proxy_port))
             self._docker_ssh_proxy_command = lambda ssh: ' '.join(
-                ssh + ssh_options_list(ssh_private_key, None
-                                      ) + ['-W', '%h:%p', f'{ssh_user}@{ip}'])
+                ssh + ssh_options_list(ssh_private_key,
+                                       None,
+                                       ssh_proxy_command=inner_proxy_command,
+                                       port=inner_proxy_port,
+                                       disable_control_master=self.
+                                       disable_control_master) +
+                ['-W', '%h:%p', f'{ssh_user}@{ip}'])
         else:
             self.ip = ip
             self.ssh_user = ssh_user

sky/utils/command_runner.pyi

@@ -142,8 +142,10 @@ class SSHCommandRunner(CommandRunner):
         ssh_user: str,
         ssh_private_key: str,
         ssh_control_name: Optional[str] = ...,
+        ssh_proxy_command: Optional[str] = ...,
         docker_user: Optional[str] = ...,
         disable_control_master: Optional[bool] = ...,
+        port_forward_execute_remote_command: Optional[bool] = ...,
     ) -> None:
         ...
 
@@ -198,6 +200,15 @@ class SSHCommandRunner(CommandRunner):
             **kwargs) -> Union[Tuple[int, str, str], int]:
         ...
 
+    def ssh_base_command(
+        self,
+        *,
+        ssh_mode: SshMode,
+        port_forward: Optional[List[Tuple[int, int]]],
+        connect_timeout: Optional[int],
+    ) -> List[str]:
+        ...
+
     def rsync(self,
               source: str,
               target: str,

sky/utils/volume.py

@@ -26,6 +26,11 @@ class VolumeType(enum.Enum):
     PVC = 'k8s-pvc'
     RUNPOD_NETWORK_VOLUME = 'runpod-network-volume'
 
+    @classmethod
+    def supported_types(cls) -> list:
+        """Return list of supported volume type values."""
+        return [vt.value for vt in cls]
+
 
 class VolumeMount:
     """Volume mount specification."""

sky/volumes/client/sdk.py

@@ -1,11 +1,12 @@
 """SDK functions for managed jobs."""
 import json
 import typing
-from typing import Any, Dict, List
+from typing import List
 
 from sky import exceptions
 from sky import sky_logging
 from sky.adaptors import common as adaptors_common
+from sky.schemas.api import responses
 from sky.server import common as server_common
 from sky.server import versions
 from sky.server.requests import payloads
@@ -116,7 +117,7 @@ def validate(volume: volume_lib.Volume) -> None:
 @usage_lib.entrypoint
 @server_common.check_server_healthy_or_start
 @annotations.client_api
-def ls() -> server_common.RequestId[List[Dict[str, Any]]]:
+def ls() -> server_common.RequestId[List[responses.VolumeRecord]]:
     """Lists all volumes.
 
     Returns:

sky/volumes/server/core.py

@@ -11,6 +11,7 @@ from sky import global_user_state
 from sky import models
 from sky import provision
 from sky import sky_logging
+from sky.schemas.api import responses
 from sky.utils import common_utils
 from sky.utils import registry
 from sky.utils import rich_utils
@@ -56,7 +57,7 @@ def volume_refresh():
                         volume_name, status=status_lib.VolumeStatus.IN_USE)
 
 
-def volume_list() -> List[Dict[str, Any]]:
+def volume_list() -> List[responses.VolumeRecord]:
     """Gets the volumes.
 
     Returns:
@@ -143,7 +144,7 @@ def volume_list() -> List[Dict[str, Any]]:
             record['name_on_cloud'] = config.name_on_cloud
             record['usedby_pods'] = usedby_pods
             record['usedby_clusters'] = usedby_clusters
-            records.append(record)
+            records.append(responses.VolumeRecord(**record))
         return records