skypilot-nightly 1.0.0.dev20250801__tar.gz → 1.0.0.dev20250804__tar.gz

{skypilot_nightly-1.0.0.dev20250801/skypilot_nightly.egg-info → skypilot_nightly-1.0.0.dev20250804}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skypilot-nightly
-Version: 1.0.0.dev20250801
+Version: 1.0.0.dev20250804
 Summary: SkyPilot: Run AI on Any Infra — Unified, Faster, Cheaper.
 Author: SkyPilot Team
 License: Apache 2.0
@@ -121,7 +121,7 @@ Requires-Dist: vastai-sdk>=0.1.12; extra == "vast"
 Provides-Extra: vsphere
 Requires-Dist: pyvmomi==8.0.1.0.2; extra == "vsphere"
 Provides-Extra: nebius
-Requires-Dist: nebius>=0.2.37; extra == "nebius"
+Requires-Dist: nebius>=0.2.47; extra == "nebius"
 Requires-Dist: awscli>=1.27.10; extra == "nebius"
 Requires-Dist: botocore>=1.29.10; extra == "nebius"
 Requires-Dist: boto3>=1.26.1; extra == "nebius"
@@ -177,7 +177,7 @@ Requires-Dist: azure-core>=1.24.0; extra == "all"
 Requires-Dist: azure-common; extra == "all"
 Requires-Dist: vastai-sdk>=0.1.12; extra == "all"
 Requires-Dist: pyvmomi==8.0.1.0.2; extra == "all"
-Requires-Dist: nebius>=0.2.37; extra == "all"
+Requires-Dist: nebius>=0.2.47; extra == "all"
 Requires-Dist: awscli>=1.27.10; extra == "all"
 Requires-Dist: botocore>=1.29.10; extra == "all"
 Requires-Dist: boto3>=1.26.1; extra == "all"
@@ -235,6 +235,7 @@ Dynamic: summary
 ----
 
 :fire: *News* :fire:
+- [Jul 2025] Run distributed **RL training for LLMs** with Verl (PPO, GRPO) on any cloud: [**example**](./llm/verl/)
 - [Jul 2025] 🎉 SkyPilot v0.10.0 released! [**blog post**](https://blog.skypilot.co/announcing-skypilot-0.10.0/), [**release notes**](https://github.com/skypilot-org/skypilot/releases/tag/v0.10.0)
 - [Jul 2025] Finetune **Llama4** on any distributed cluster/cloud: [**example**](./llm/llama-4-finetuning/)
 - [Jul 2025] Two-part blog series, `The Evolution of AI Job Orchestration`: (1) [Running AI jobs on GPU Neoclouds](https://blog.skypilot.co/ai-job-orchestration-pt1-gpu-neoclouds/), (2) [The AI-Native Control Plane & Orchestration that Finally Works for ML](https://blog.skypilot.co/ai-job-orchestration-pt2-ai-control-plane/)

{skypilot_nightly-1.0.0.dev20250801 → skypilot_nightly-1.0.0.dev20250804}/README.md

@@ -39,6 +39,7 @@
 ----
 
 :fire: *News* :fire:
+- [Jul 2025] Run distributed **RL training for LLMs** with Verl (PPO, GRPO) on any cloud: [**example**](./llm/verl/)
 - [Jul 2025] 🎉 SkyPilot v0.10.0 released! [**blog post**](https://blog.skypilot.co/announcing-skypilot-0.10.0/), [**release notes**](https://github.com/skypilot-org/skypilot/releases/tag/v0.10.0)
 - [Jul 2025] Finetune **Llama4** on any distributed cluster/cloud: [**example**](./llm/llama-4-finetuning/)
 - [Jul 2025] Two-part blog series, `The Evolution of AI Job Orchestration`: (1) [Running AI jobs on GPU Neoclouds](https://blog.skypilot.co/ai-job-orchestration-pt1-gpu-neoclouds/), (2) [The AI-Native Control Plane & Orchestration that Finally Works for ML](https://blog.skypilot.co/ai-job-orchestration-pt2-ai-control-plane/)

{skypilot_nightly-1.0.0.dev20250801 → skypilot_nightly-1.0.0.dev20250804}/sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '45dfcb679d71b0fb8abe30470b1d8c6b6c33ef73'
+_SKYPILOT_COMMIT_SHA = '641a8e762a1d86274420f7a9d9cea6657b5be5b4'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250801'
+__version__ = '1.0.0.dev20250804'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

{skypilot_nightly-1.0.0.dev20250801 → skypilot_nightly-1.0.0.dev20250804}/sky/backends/backend_utils.py

@@ -38,6 +38,7 @@ from sky.provision import instance_setup
 from sky.provision.kubernetes import utils as kubernetes_utils
 from sky.serve import serve_utils
 from sky.server.requests import requests as requests_lib
+from sky.skylet import autostop_lib
 from sky.skylet import constants
 from sky.usage import usage_lib
 from sky.utils import cluster_utils
@@ -922,7 +923,10 @@ def write_cluster_config(
             cluster_config_overrides=cluster_config_overrides,
             cloud=cloud,
             context=region.name)
-        kubernetes_utils.combine_metadata_fields(tmp_yaml_path, region.name)
+        kubernetes_utils.combine_metadata_fields(
+            tmp_yaml_path,
+            cluster_config_overrides=cluster_config_overrides,
+            context=region.name)
         yaml_obj = common_utils.read_yaml(tmp_yaml_path)
         pod_config: Dict[str, Any] = yaml_obj['available_node_types'][
             'ray_head_default']['node_config']
@@ -2238,7 +2242,11 @@ def _update_cluster_status(cluster_name: str) -> Optional[Dict[str, Any]]:
                     success = True
                     reset_local_autostop = True
                     try:
-                        backend.set_autostop(handle, -1, stream_logs=False)
+                        backend.set_autostop(
+                            handle,
+                            -1,
+                            autostop_lib.DEFAULT_AUTOSTOP_WAIT_FOR,
+                            stream_logs=False)
                     except exceptions.CommandError as e:
                         success = False
                         if e.returncode == 255:

{skypilot_nightly-1.0.0.dev20250801 → skypilot_nightly-1.0.0.dev20250804}/sky/backends/cloud_vm_ray_backend.py

@@ -4650,6 +4650,7 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
     def set_autostop(self,
                      handle: CloudVmRayResourceHandle,
                      idle_minutes_to_autostop: Optional[int],
+                     wait_for: Optional[autostop_lib.AutostopWaitFor],
                      down: bool = False,
                      stream_logs: bool = True) -> None:
         # The core.autostop() function should have already checked that the
@@ -4697,7 +4698,7 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
             assert (handle.launched_resources is not None and
                     handle.launched_resources.cloud is not None), handle
             code = autostop_lib.AutostopCodeGen.set_autostop(
-                idle_minutes_to_autostop, self.NAME, down)
+                idle_minutes_to_autostop, self.NAME, wait_for, down)
             returncode, _, stderr = self.run_on_head(handle,
                                                      code,
                                                      require_outputs=True,

{skypilot_nightly-1.0.0.dev20250801 → skypilot_nightly-1.0.0.dev20250804}/sky/catalog/data_fetchers/fetch_nebius.py

@@ -31,17 +31,19 @@ class PresetInfo:
     Attributes:
         region (str): The geographical region where the preset is available.
         fullname (str): The full name of the preset, a combination of platform
-        and preset name.
+            and preset name.
         name (str): The name of the preset.
         platform_name (str): The name of the platform the preset belongs to.
         gpu (int): The number of GPUs in the preset.
         vcpu (int): The number of virtual CPUs in the preset.
         memory_gib (int): The amount of memory in GiB in the preset.
         accelerator_manufacturer (str | None): The manufacturer of the
-        accelerator (e.g., "NVIDIA"), or None if no accelerator.
+            accelerator (e.g., "NVIDIA"), or None if no accelerator.
         accelerator_name (str | None): The name of the accelerator
-        (e.g., "H100"), or None if no accelerator.
+            (e.g., "H100"), or None if no accelerator.
         price_hourly (decimal.Decimal): The hourly price of the preset.
+        spot_price (decimal.Decimal): The spot (preemptible) price
+            of the preset.
     """
 
     region: str
@@ -54,6 +56,7 @@ class PresetInfo:
     accelerator_manufacturer: Optional[str]
     accelerator_name: Optional[str]
     price_hourly: decimal.Decimal
+    spot_price: decimal.Decimal
 
 
 def _format_decimal(value: decimal.Decimal) -> str:
@@ -66,7 +69,7 @@ def _format_decimal(value: decimal.Decimal) -> str:
     Returns:
         str: The formatted string representation of the decimal.
     """
-    formatted_value = f'{value:f}'.rstrip('0').rstrip('.')
+    formatted_value = f'{value:f}'
     integer_part, decimal_part = formatted_value.split(
         '.') if '.' in formatted_value else (formatted_value, '')
     if len(decimal_part) < 2:
@@ -111,20 +114,38 @@ def _estimate_platforms(platforms: List[Any], parent_id: str,
                             preset=preset.name,
                         )),
                 ))
-
             price_request = billing().EstimateBatchRequest(
                 resource_specs=[estimate_spec])
+
+            # Form the specification for the spot price request
+            spot_estimate_spec = billing().ResourceSpec(
+                compute_instance_spec=compute().CreateInstanceRequest(
+                    metadata=nebius_common().ResourceMetadata(
+                        parent_id=parent_id,),
+                    spec=compute().InstanceSpec(
+                        resources=compute().ResourcesSpec(
+                            platform=platform_name,
+                            preset=preset.name,
+                        ),
+                        preemptible=compute().PreemptibleSpec(priority=1),
+                    ),
+                ))
+            spot_price_request = billing().EstimateBatchRequest(
+                resource_specs=[spot_estimate_spec])
+
             # Start future for each preset
             futures.append((
                 platform,
                 preset,
                 calculator_service.estimate_batch(price_request,
                                                   timeout=TIMEOUT),
+                calculator_service.estimate_batch(spot_price_request,
+                                                  timeout=TIMEOUT),
             ))
 
     # wait all futures to complete and collect results
     result = []
-    for platform, preset, future in futures:
+    for platform, preset, future, future_spot in futures:
         platform_name = platform.metadata.name
         result.append(
             PresetInfo(
@@ -141,6 +162,8 @@ def _estimate_platforms(platforms: List[Any], parent_id: str,
                 if platform_name.startswith('gpu-') else '',
                 price_hourly=decimal.Decimal(
                     future.wait().hourly_cost.general.total.cost),
+                spot_price=decimal.Decimal(
+                    future_spot.wait().hourly_cost.general.total.cost),
             ))
 
     return result
@@ -196,7 +219,8 @@ def _write_preset_prices(presets: List[PresetInfo], output_file: str) -> None:
                 'Price': _format_decimal(preset.price_hourly),
                 'Region': preset.region,
                 'GpuInfo': gpu_info,
-                'SpotPrice': '',
+                'SpotPrice': _format_decimal(preset.spot_price)
+                             if preset.spot_price else '',
             })
 
 

{skypilot_nightly-1.0.0.dev20250801 → skypilot_nightly-1.0.0.dev20250804}/sky/client/cli/command.py

@@ -62,8 +62,8 @@ from sky.provision.kubernetes import constants as kubernetes_constants
 from sky.provision.kubernetes import utils as kubernetes_utils
 from sky.server import common as server_common
 from sky.server import constants as server_constants
-from sky.server import versions
 from sky.server.requests import requests
+from sky.skylet import autostop_lib
 from sky.skylet import constants
 from sky.skylet import job_lib
 from sky.usage import usage_lib
@@ -1012,11 +1012,11 @@ def _handle_infra_cloud_region_zone_options(infra: Optional[str],
     required=False,
     help=('Automatically stop the cluster after this many minutes '
           'of idleness, i.e., no running or pending jobs in the cluster\'s job '
-          'queue. Idleness gets reset whenever setting-up/running/pending jobs '
-          'are found in the job queue. '
+          'queue. Idleness gets reset depending on the ``--wait-for`` flag. '
           'Setting this flag is equivalent to '
           'running ``sky launch -d ...`` and then ``sky autostop -i <minutes>``'
           '. If not set, the cluster will not be autostopped.'))
+@flags.wait_for_option('idle-minutes-to-autostop')
 @click.option(
     '--down',
     default=False,
@@ -1102,6 +1102,7 @@ def launch(
     network_tier: Optional[str],
     ports: Tuple[str, ...],
     idle_minutes_to_autostop: Optional[int],
+    wait_for: Optional[str],
     down: bool,  # pylint: disable=redefined-outer-name
     retry_until_up: bool,
     yes: bool,
@@ -1196,6 +1197,8 @@ def launch(
         cluster_name=cluster,
         backend=backend,
         idle_minutes_to_autostop=idle_minutes_to_autostop,
+        wait_for=autostop_lib.AutostopWaitFor.from_str(wait_for)
+        if wait_for is not None else None,
         down=down,
         retry_until_up=retry_until_up,
         no_setup=no_setup,
@@ -1828,9 +1831,6 @@ def status(verbose: bool, refresh: bool, ip: bool, endpoints: bool,
     show_endpoints = endpoints or endpoint is not None
     show_single_endpoint = endpoint is not None
     show_services = show_services and not any([clusters, ip, endpoints])
-    remote_api_version = versions.get_remote_api_version()
-    if remote_api_version is None or remote_api_version < 12:
-        show_pools = False
 
     query_clusters: Optional[List[str]] = None if not clusters else clusters
     refresh_mode = common.StatusRefreshMode.NONE
@@ -1882,7 +1882,11 @@ def status(verbose: bool, refresh: bool, ip: bool, endpoints: bool,
         return serve_lib.status(service_names=None)
 
     def submit_pools() -> Optional[str]:
-        return managed_jobs.pool_status(pool_names=None)
+        try:
+            return managed_jobs.pool_status(pool_names=None)
+        except exceptions.APINotSupportedError as e:
+            logger.debug(f'Pools are not supported in the remote server: {e}')
+            return None
 
     def submit_workspace() -> Optional[str]:
         try:
@@ -2005,7 +2009,7 @@ def status(verbose: bool, refresh: bool, ip: bool, endpoints: bool,
                 controller_utils.Controllers.JOBS_CONTROLLER.value.
                 in_progress_hint(False).format(job_info=job_info))
 
-    if show_pools:
+    if show_pools and pool_status_request_id:
         num_pools = None
         if managed_jobs_query_interrupted:
             msg = 'KeyboardInterrupt'
@@ -2549,6 +2553,7 @@ def stop(
               required=False,
               help=('Set the idle minutes before autostopping the cluster. '
                     'See the doc above for detailed semantics.'))
+@flags.wait_for_option('idle-minutes')
 @click.option(
     '--cancel',
     default=False,
@@ -2571,6 +2576,7 @@ def autostop(
     all: bool,  # pylint: disable=redefined-builtin
     all_users: bool,
     idle_minutes: Optional[int],
+    wait_for: Optional[str],
     cancel: bool,  # pylint: disable=redefined-outer-name
     down: bool,  # pylint: disable=redefined-outer-name
     yes: bool,
@@ -2580,8 +2586,7 @@ def autostop(
     """Schedule an autostop or autodown for cluster(s).
 
     Autostop/autodown will automatically stop or teardown a cluster when it
-    becomes idle for a specified duration.  Idleness means there are no
-    in-progress (pending/running) jobs in a cluster's job queue.
+    becomes idle for a specified duration.
 
     CLUSTERS are the names (or glob patterns) of the clusters to stop. If both
     CLUSTERS and ``--all`` are supplied, the latter takes precedence.
@@ -2594,6 +2599,11 @@ def autostop(
 
     - An autostop idle time is set.
 
+    - An SSH session is active (To disable this, set ``--wait-for jobs``).
+
+    To disable the idleness timer completely and set a hard time limit, set
+    ``--wait-for none``.
+
     Example 1: say a cluster with autostop set to 2 hours has been idle for 1
     hour, then autostop is reset to 30 minutes. The cluster will not be
     immediately autostopped. Instead, the idleness timer restarts counting
@@ -2614,6 +2624,9 @@ def autostop(
         # Cancel autostop for a specific cluster.
         sky autostop cluster_name --cancel
         \b
+        # Autostop this cluster after 60 minutes, regardless of activity.
+        sky autostop cluster_name -i 60 --wait-for none
+        \b
         # Autodown this cluster after 60 minutes of idleness.
         sky autostop cluster_name -i 60 --down
     """
@@ -2625,13 +2638,16 @@ def autostop(
         idle_minutes = -1
     elif idle_minutes is None:
         idle_minutes = 5
-    _down_or_stop_clusters(clusters,
-                           apply_to_all=all,
-                           all_users=all_users,
-                           down=down,
-                           no_confirm=yes,
-                           idle_minutes_to_autostop=idle_minutes,
-                           async_call=async_call)
+    _down_or_stop_clusters(
+        clusters,
+        apply_to_all=all,
+        all_users=all_users,
+        down=down,
+        no_confirm=yes,
+        idle_minutes_to_autostop=idle_minutes,
+        wait_for=autostop_lib.AutostopWaitFor.from_str(wait_for)
+        if wait_for is not None else None,
+        async_call=async_call)
 
 
 @cli.command(cls=_DocumentedCodeCommand)
@@ -2650,11 +2666,11 @@ def autostop(
     required=False,
     help=('Automatically stop the cluster after this many minutes '
           'of idleness, i.e., no running or pending jobs in the cluster\'s job '
-          'queue. Idleness gets reset whenever setting-up/running/pending jobs '
-          'are found in the job queue. '
+          'queue. Idleness gets reset depending on the ``--wait-for`` flag. '
           'Setting this flag is equivalent to '
           'running ``sky launch -d ...`` and then ``sky autostop -i <minutes>``'
           '. If not set, the cluster will not be autostopped.'))
+@flags.wait_for_option('idle-minutes-to-autostop')
 @click.option(
     '--down',
     default=False,
@@ -2693,6 +2709,7 @@ def start(
     all: bool,
     yes: bool,
     idle_minutes_to_autostop: Optional[int],
+    wait_for: Optional[str],
     down: bool,  # pylint: disable=redefined-outer-name
     retry_until_up: bool,
     force: bool,
@@ -2852,6 +2869,8 @@ def start(
     request_ids = subprocess_utils.run_in_parallel(
         lambda name: sdk.start(name,
                                idle_minutes_to_autostop,
+                               autostop_lib.AutostopWaitFor.from_str(wait_for)
+                               if wait_for is not None else None,
                                retry_until_up,
                                down=down,
                                force=force), to_start)
@@ -3081,6 +3100,7 @@ def _down_or_stop_clusters(
         no_confirm: bool = True,
         purge: bool = False,
         idle_minutes_to_autostop: Optional[int] = None,
+        wait_for: Optional[autostop_lib.AutostopWaitFor] = None,
         async_call: bool = False) -> None:
     """Tears down or (auto-)stops a cluster (or all clusters).
 
@@ -3098,6 +3118,7 @@ def _down_or_stop_clusters(
         purge: If True, forcefully remove the clusters from the cluster table.
         idle_minutes_to_autostop: The number of minutes to wait before
             automatically stopping the cluster.
+        wait_for: Determines the condition for resetting the idleness timer.
         async_call: If True, send the request asynchronously.
     """
     if down:
@@ -3241,7 +3262,8 @@ def _down_or_stop_clusters(
         success_progress = False
         if idle_minutes_to_autostop is not None:
             try:
-                request_id = sdk.autostop(name, idle_minutes_to_autostop, down)
+                request_id = sdk.autostop(name, idle_minutes_to_autostop,
+                                          wait_for, down)
                 request_ids.append(request_id)
                 _async_call_or_wait(
                     request_id, async_call,

{skypilot_nightly-1.0.0.dev20250801 → skypilot_nightly-1.0.0.dev20250804}/sky/client/cli/flags.py

@@ -7,6 +7,7 @@ import click
 import dotenv
 
 from sky import skypilot_config
+from sky.skylet import autostop_lib
 from sky.utils import resources_utils
 
 
@@ -340,3 +341,17 @@ def all_users_option(helptext: Optional[str] = None):
                             help=helptext)(func)
 
     return return_option_decorator
+
+
+def wait_for_option(pair: str):
+    """A decorator for the --wait-for option."""
+
+    def return_option_decorator(func):
+        return click.option(
+            '--wait-for',
+            type=click.Choice(autostop_lib.AutostopWaitFor.supported_modes()),
+            default=None,
+            required=False,
+            help=autostop_lib.AutostopWaitFor.cli_help_message(pair=pair))(func)
+
+    return return_option_decorator

{skypilot_nightly-1.0.0.dev20250801 → skypilot_nightly-1.0.0.dev20250804}/sky/client/sdk.py

@@ -10,19 +10,14 @@ Usage example:
     statuses = sky.get(request_id)
 
 """
-import base64
-import binascii
 from http import cookiejar
 import json
 import logging
 import os
-import pathlib
 import subprocess
-import time
 import typing
 from typing import Any, Dict, List, Optional, Tuple, Union
 from urllib import parse as urlparse
-import webbrowser
 
 import click
 import colorama
@@ -37,8 +32,10 @@ from sky.client import common as client_common
 from sky.client import oauth as oauth_lib
 from sky.server import common as server_common
 from sky.server import rest
+from sky.server import versions
 from sky.server.requests import payloads
 from sky.server.requests import requests as requests_lib
+from sky.skylet import autostop_lib
 from sky.skylet import constants
 from sky.usage import usage_lib
 from sky.utils import admin_policy_utils
@@ -57,7 +54,12 @@ from sky.utils import ux_utils
 from sky.utils.kubernetes import ssh_utils
 
 if typing.TYPE_CHECKING:
+    import base64
+    import binascii
     import io
+    import pathlib
+    import time
+    import webbrowser
 
     import psutil
     import requests
@@ -65,6 +67,14 @@ if typing.TYPE_CHECKING:
     import sky
     from sky import backends
 else:
+    # only used in api_login()
+    base64 = adaptors_common.LazyImport('base64')
+    binascii = adaptors_common.LazyImport('binascii')
+    pathlib = adaptors_common.LazyImport('pathlib')
+    time = adaptors_common.LazyImport('time')
+    # only used in dashboard() and api_login()
+    webbrowser = adaptors_common.LazyImport('webbrowser')
+    # only used in api_stop()
     psutil = adaptors_common.LazyImport('psutil')
 
 logger = sky_logging.init_logger(__name__)
@@ -375,6 +385,7 @@ def launch(
     cluster_name: Optional[str] = None,
     retry_until_up: bool = False,
     idle_minutes_to_autostop: Optional[int] = None,
+    wait_for: Optional[autostop_lib.AutostopWaitFor] = None,
     dryrun: bool = False,
     down: bool = False,  # pylint: disable=redefined-outer-name
     backend: Optional['backends.Backend'] = None,
@@ -424,6 +435,15 @@ def launch(
             ``sky.autostop(idle_minutes=<minutes>)``. If set, the autostop
             config specified in the task' resources will be overridden by
             this parameter.
+        wait_for: determines the condition for resetting the idleness timer.
+            This option works in conjunction with ``idle_minutes_to_autostop``.
+            Choices:
+
+            1. "jobs_and_ssh" (default) - Wait for all jobs to complete
+               AND all SSH sessions to disconnect.
+            2. "jobs" - Wait for all jobs to complete.
+            3. "none" - Stop immediately after idle time expires,
+               regardless of running jobs or SSH connections.
         dryrun: if True, do not actually launch the cluster.
         down: Tear down the cluster after all jobs finish (successfully or
             abnormally). If --idle-minutes-to-autostop is also set, the
@@ -487,12 +507,27 @@ def launch(
             raise NotImplementedError('clone_disk_from is not implemented yet. '
                                       'Please contact the SkyPilot team if you '
                                       'need this feature at slack.skypilot.co.')
+
+    remote_api_version = versions.get_remote_api_version()
+    if wait_for is not None and (remote_api_version is None or
+                                 remote_api_version < 13):
+        logger.warning('wait_for is not supported in your API server. '
+                       'Please upgrade to a newer API server to use it.')
+
     dag = dag_utils.convert_entrypoint_to_dag(task)
     # Override the autostop config from command line flags to task YAML.
     for task in dag.tasks:
         for resource in task.resources:
-            resource.override_autostop_config(
-                down=down, idle_minutes=idle_minutes_to_autostop)
+            if remote_api_version is None or remote_api_version < 13:
+                # An older server would not recognize the wait_for field
+                # in the schema, so we need to omit it.
+                resource.override_autostop_config(
+                    down=down, idle_minutes=idle_minutes_to_autostop)
+            else:
+                resource.override_autostop_config(
+                    down=down,
+                    idle_minutes=idle_minutes_to_autostop,
+                    wait_for=wait_for)
             if resource.autostop_config is not None:
                 # For backward-compatbility, get the final autostop config for
                 # admin policy.
@@ -825,6 +860,7 @@ def download_logs(cluster_name: str,
 def start(
     cluster_name: str,
     idle_minutes_to_autostop: Optional[int] = None,
+    wait_for: Optional[autostop_lib.AutostopWaitFor] = None,
     retry_until_up: bool = False,
     down: bool = False,  # pylint: disable=redefined-outer-name
     force: bool = False,
@@ -851,6 +887,15 @@ def start(
             flag is equivalent to running ``sky.launch()`` and then
             ``sky.autostop(idle_minutes=<minutes>)``. If not set, the
             cluster will not be autostopped.
+        wait_for: determines the condition for resetting the idleness timer.
+            This option works in conjunction with ``idle_minutes_to_autostop``.
+            Choices:
+
+            1. "jobs_and_ssh" (default) - Wait for all jobs to complete
+               AND all SSH sessions to disconnect.
+            2. "jobs" - Wait for all jobs to complete.
+            3. "none" - Stop immediately after idle time expires,
+               regardless of running jobs or SSH connections.
         retry_until_up: whether to retry launching the cluster until it is
             up.
         down: Autodown the cluster: tear down the cluster after specified
@@ -879,9 +924,16 @@ def start(
         sky.exceptions.ClusterOwnerIdentitiesMismatchError: if the cluster to
             restart was launched by a different user.
     """
+    remote_api_version = versions.get_remote_api_version()
+    if wait_for is not None and (remote_api_version is None or
+                                 remote_api_version < 13):
+        logger.warning('wait_for is not supported in your API server. '
+                       'Please upgrade to a newer API server to use it.')
+
     body = payloads.StartBody(
         cluster_name=cluster_name,
         idle_minutes_to_autostop=idle_minutes_to_autostop,
+        wait_for=wait_for,
         retry_until_up=retry_until_up,
         down=down,
         force=force,
@@ -982,9 +1034,10 @@ def stop(cluster_name: str, purge: bool = False) -> server_common.RequestId:
 @server_common.check_server_healthy_or_start
 @annotations.client_api
 def autostop(
-    cluster_name: str,
-    idle_minutes: int,
-    down: bool = False  # pylint: disable=redefined-outer-name
+        cluster_name: str,
+        idle_minutes: int,
+        wait_for: Optional[autostop_lib.AutostopWaitFor] = None,
+        down: bool = False,  # pylint: disable=redefined-outer-name
 ) -> server_common.RequestId:
     """Schedules an autostop/autodown for a cluster.
 
@@ -1015,6 +1068,15 @@ def autostop(
         idle_minutes: the number of minutes of idleness (no pending/running
             jobs) after which the cluster will be stopped automatically. Setting
             to a negative number cancels any autostop/autodown setting.
+        wait_for: determines the condition for resetting the idleness timer.
+            This option works in conjunction with ``idle_minutes``.
+            Choices:
+
+            1. "jobs_and_ssh" (default) - Wait for all jobs to complete
+               AND all SSH sessions to disconnect.
+            2. "jobs" - Wait for all jobs to complete.
+            3. "none" - Stop immediately after idle time expires,
+               regardless of running jobs or SSH connections.
         down: if true, use autodown (tear down the cluster; non-restartable),
             rather than autostop (restartable).
 
@@ -1034,9 +1096,16 @@ def autostop(
         sky.exceptions.CloudUserIdentityError: if we fail to get the current
             user identity.
     """
+    remote_api_version = versions.get_remote_api_version()
+    if wait_for is not None and (remote_api_version is None or
+                                 remote_api_version < 13):
+        logger.warning('wait_for is not supported in your API server. '
+                       'Please upgrade to a newer API server to use it.')
+
     body = payloads.AutostopBody(
         cluster_name=cluster_name,
         idle_minutes=idle_minutes,
+        wait_for=wait_for,
         down=down,
     )
     response = server_common.make_authenticated_request(
@@ -2343,6 +2412,7 @@ def api_login(endpoint: Optional[str] = None,
     _save_config_updates(endpoint=endpoint)
     dashboard_url = server_common.get_dashboard_url(endpoint)
 
+    server_common.get_api_server_status.cache_clear()
     # After successful authentication, check server health again to get user
     # identity
     server_status, final_api_server_info = server_common.check_server_healthy(

{skypilot_nightly-1.0.0.dev20250801 → skypilot_nightly-1.0.0.dev20250804}/sky/client/sdk.pyi

@@ -14,6 +14,7 @@ from sky import skypilot_config as skypilot_config
 from sky.server import common as server_common
 from sky.server import rest as rest
 from sky.server.requests import payloads as payloads
+from sky.skylet import autostop_lib as autostop_lib
 from sky.skylet import constants as constants
 from sky.usage import usage_lib as usage_lib
 from sky.utils import admin_policy_utils as admin_policy_utils
@@ -104,6 +105,7 @@ def launch(task: Union['sky.Task', 'sky.Dag'],
            cluster_name: Optional[str] = ...,
            retry_until_up: bool = ...,
            idle_minutes_to_autostop: Optional[int] = ...,
+           wait_for: Optional[autostop_lib.AutostopWaitFor] = ...,
            dryrun: bool = ...,
            down: bool = ...,
            backend: Optional['backends.Backend'] = ...,
@@ -142,6 +144,7 @@ def download_logs(cluster_name: str,
 
 def start(cluster_name: str,
           idle_minutes_to_autostop: Optional[int] = ...,
+          wait_for: Optional[autostop_lib.AutostopWaitFor] = ...,
           retry_until_up: bool = ...,
           down: bool = ...,
           force: bool = ...) -> server_common.RequestId:
@@ -158,6 +161,7 @@ def stop(cluster_name: str, purge: bool = ...) -> server_common.RequestId:
 
 def autostop(cluster_name: str,
              idle_minutes: int,
+             wait_for: Optional[autostop_lib.AutostopWaitFor] = ...,
              down: bool = ...) -> server_common.RequestId:
     ...