skypilot-nightly 1.0.0.dev20250801__py3-none-any.whl → 1.0.0.dev20250804__py3-none-any.whl

sky/execution.py

@@ -15,6 +15,7 @@ from sky import global_user_state
 from sky import optimizer
 from sky import sky_logging
 from sky.backends import backend_utils
+from sky.skylet import autostop_lib
 from sky.usage import usage_lib
 from sky.utils import admin_policy_utils
 from sky.utils import common
@@ -309,11 +310,13 @@ def _execute_dag(
 
         idle_minutes_to_autostop: Optional[int] = None
         down = False
+        wait_for: Optional[autostop_lib.AutostopWaitFor] = None
         if resource_autostop_config is not None:
             if resource_autostop_config.enabled:
                 idle_minutes_to_autostop = (
                     resource_autostop_config.idle_minutes)
                 down = resource_autostop_config.down
+                wait_for = resource_autostop_config.wait_for
             else:
                 # Autostop is explicitly disabled, so cancel it if it's
                 # already set.
@@ -450,9 +453,8 @@ def _execute_dag(
             if idle_minutes_to_autostop is not None:
                 assert isinstance(backend, backends.CloudVmRayBackend)
                 assert isinstance(handle, backends.CloudVmRayResourceHandle)
-                backend.set_autostop(handle,
-                                     idle_minutes_to_autostop,
-                                     down=down)
+                backend.set_autostop(handle, idle_minutes_to_autostop, wait_for,
+                                     down)
 
         if Stage.EXEC in stages:
             try:

sky/jobs/client/sdk.py

@@ -2,11 +2,11 @@
 import json
 import typing
 from typing import Dict, List, Optional, Union
-import webbrowser
 
 import click
 
 from sky import sky_logging
+from sky.adaptors import common as adaptors_common
 from sky.client import common as client_common
 from sky.client import sdk
 from sky.serve.client import impl
@@ -23,9 +23,13 @@ from sky.utils import dag_utils
 
 if typing.TYPE_CHECKING:
     import io
+    import webbrowser
 
     import sky
     from sky.serve import serve_utils
+else:
+    # only used in dashboard()
+    webbrowser = adaptors_common.LazyImport('webbrowser')
 
 logger = sky_logging.init_logger(__name__)
 

sky/provision/kubernetes/utils.py

@@ -2642,6 +2642,7 @@ def combine_pod_config_fields(
 
 
 def combine_metadata_fields(cluster_yaml_path: str,
+                            cluster_config_overrides: Dict[str, Any],
                             context: Optional[str] = None) -> None:
     """Updates the metadata for all Kubernetes objects created by SkyPilot with
     fields from the ~/.sky/config.yaml's kubernetes.custom_metadata dict.
@@ -2652,12 +2653,25 @@ def combine_metadata_fields(cluster_yaml_path: str,
     with open(cluster_yaml_path, 'r', encoding='utf-8') as f:
         yaml_content = f.read()
     yaml_obj = yaml.safe_load(yaml_content)
+
+    # Get custom_metadata from global config
     custom_metadata = skypilot_config.get_effective_region_config(
         cloud='kubernetes',
         region=context,
         keys=('custom_metadata',),
         default_value={})
 
+    # Get custom_metadata from task-level config overrides
+    override_custom_metadata = config_utils.get_cloud_config_value_from_dict(
+        dict_config=cluster_config_overrides,
+        cloud='kubernetes',
+        region=context,
+        keys=('custom_metadata',),
+        default_value={})
+
+    # Merge task-level overrides with global config
+    config_utils.merge_k8s_configs(custom_metadata, override_custom_metadata)
+
     # List of objects in the cluster YAML to be updated
     combination_destinations = [
         # Service accounts
@@ -2679,17 +2693,33 @@ def combine_metadata_fields(cluster_yaml_path: str,
     common_utils.dump_yaml(cluster_yaml_path, yaml_obj)
 
 
-def merge_custom_metadata(original_metadata: Dict[str, Any],
-                          context: Optional[str] = None) -> None:
+def merge_custom_metadata(
+        original_metadata: Dict[str, Any],
+        context: Optional[str] = None,
+        cluster_config_overrides: Optional[Dict[str, Any]] = None) -> None:
     """Merges original metadata with custom_metadata from config
 
     Merge is done in-place, so return is not required
     """
+    # Get custom_metadata from global config
     custom_metadata = skypilot_config.get_effective_region_config(
         cloud='kubernetes',
         region=context,
         keys=('custom_metadata',),
         default_value={})
+
+    # Get custom_metadata from task-level config overrides if available
+    if cluster_config_overrides is not None:
+        override_custom_metadata = config_utils.get_cloud_config_value_from_dict(
+            dict_config=cluster_config_overrides,
+            cloud='kubernetes',
+            region=context,
+            keys=('custom_metadata',),
+            default_value={})
+        # Merge task-level overrides with global config
+        config_utils.merge_k8s_configs(custom_metadata,
+                                       override_custom_metadata)
+
     config_utils.merge_k8s_configs(original_metadata, custom_metadata)
 
 

sky/resources.py

@@ -20,6 +20,7 @@ from sky.provision import docker_utils
 from sky.provision.gcp import constants as gcp_constants
 from sky.provision.kubernetes import utils as kubernetes_utils
 from sky.provision.nebius import constants as nebius_constants
+from sky.skylet import autostop_lib
 from sky.skylet import constants
 from sky.utils import accelerator_registry
 from sky.utils import annotations
@@ -69,14 +70,18 @@ class AutostopConfig:
     # flags.
     idle_minutes: int = 0
     down: bool = False
+    wait_for: Optional[autostop_lib.AutostopWaitFor] = None
 
     def to_yaml_config(self) -> Union[Literal[False], Dict[str, Any]]:
         if not self.enabled:
             return False
-        return {
+        config: Dict[str, Any] = {
             'idle_minutes': self.idle_minutes,
             'down': self.down,
         }
+        if self.wait_for is not None:
+            config['wait_for'] = self.wait_for.value
+        return config
 
     @classmethod
     def from_yaml_config(
@@ -104,6 +109,9 @@ class AutostopConfig:
                 autostop_config.idle_minutes = config['idle_minutes']
             if 'down' in config:
                 autostop_config.down = config['down']
+            if 'wait_for' in config:
+                autostop_config.wait_for = (
+                    autostop_lib.AutostopWaitFor.from_str(config['wait_for']))
             return autostop_config
 
         return None
@@ -958,15 +966,18 @@ class Resources:
             valid_volumes.append(volume)
         self._volumes = valid_volumes
 
-    def override_autostop_config(self,
-                                 down: bool = False,
-                                 idle_minutes: Optional[int] = None) -> None:
+    def override_autostop_config(
+            self,
+            down: bool = False,
+            idle_minutes: Optional[int] = None,
+            wait_for: Optional[autostop_lib.AutostopWaitFor] = None) -> None:
         """Override autostop config to the resource.
 
         Args:
             down: If true, override the autostop config to use autodown.
             idle_minutes: If not None, override the idle minutes to autostop or
                 autodown.
+            wait_for: If not None, override the wait mode.
         """
         if not down and idle_minutes is None:
             return
@@ -976,6 +987,8 @@ class Resources:
             self._autostop_config.down = down
         if idle_minutes is not None:
             self._autostop_config.idle_minutes = idle_minutes
+        if wait_for is not None:
+            self._autostop_config.wait_for = wait_for
 
     def is_launchable(self) -> bool:
         """Returns whether the resource is launchable."""

sky/server/constants.py

@@ -10,7 +10,7 @@ from sky.skylet import constants
 # based on version info is needed.
 # For more details and code guidelines, refer to:
 # https://docs.skypilot.co/en/latest/developers/CONTRIBUTING.html#backward-compatibility-guidelines
-API_VERSION = 12
+API_VERSION = 13
 
 # The minimum peer API version that the code should still work with.
 # Notes (dev):

sky/server/requests/payloads.py

@@ -33,6 +33,7 @@ from sky import sky_logging
 from sky import skypilot_config
 from sky.adaptors import common as adaptors_common
 from sky.server import common
+from sky.skylet import autostop_lib
 from sky.skylet import constants
 from sky.usage import constants as usage_constants
 from sky.usage import usage_lib
@@ -312,6 +313,7 @@ class StartBody(RequestBody):
     """The request body for the start endpoint."""
     cluster_name: str
     idle_minutes_to_autostop: Optional[int] = None
+    wait_for: Optional[autostop_lib.AutostopWaitFor] = None
     retry_until_up: bool = False
     down: bool = False
     force: bool = False
@@ -321,6 +323,7 @@ class AutostopBody(RequestBody):
     """The request body for the autostop endpoint."""
     cluster_name: str
     idle_minutes: int
+    wait_for: Optional[autostop_lib.AutostopWaitFor] = None
     down: bool = False
 
 

sky/setup_files/dependencies.py

@@ -177,7 +177,7 @@ extras_require: Dict[str, List[str]] = {
         # 'vsphere-automation-sdk @ git+https://github.com/vmware/vsphere-automation-sdk-python.git@v8.0.1.0' pylint: disable=line-too-long
     ],
     'nebius': [
-        'nebius>=0.2.37',
+        'nebius>=0.2.47',
     ] + aws_dependencies,
     'hyperbolic': [],  # No dependencies needed for hyperbolic
     'server': server_dependencies,

sky/skylet/autostop_lib.py

@@ -1,6 +1,8 @@
 """Autostop utilities."""
+import enum
 import pickle
 import shlex
+import subprocess
 import time
 import typing
 from typing import List, Optional
@@ -10,6 +12,7 @@ from sky.adaptors import common as adaptors_common
 from sky.skylet import configs
 from sky.skylet import constants
 from sky.utils import message_utils
+from sky.utils import ux_utils
 
 if typing.TYPE_CHECKING:
     import psutil
@@ -30,6 +33,55 @@ _AUTOSTOP_LAST_ACTIVE_TIME = 'autostop_last_active_time'
 _AUTOSTOP_INDICATOR = 'autostop_indicator'
 
 
+class AutostopWaitFor(enum.Enum):
+    """Enum for the Autostop behaviour.
+
+    JOBS: Wait for jobs to finish.
+    JOBS_AND_SSH: Wait for jobs to finish and all SSH sessions to be closed.
+    NONE: Unconditionally stop the cluster after the idle time.
+    """
+    JOBS_AND_SSH = 'jobs_and_ssh'
+    JOBS = 'jobs'
+    NONE = 'none'
+
+    @classmethod
+    def supported_modes(cls) -> List[str]:
+        return [mode.value for mode in cls]
+
+    @classmethod
+    def cli_help_message(cls, pair: str) -> str:
+        return f"""\
+Determines the condition for resetting the idleness timer.
+This option works in conjunction with ``--{pair}``. Options:
+
+\b
+1. ``jobs_and_ssh`` (default): Wait for all jobs to complete AND all SSH
+sessions to disconnect.
+2. ``jobs``: Wait for all jobs to complete.
+3. ``none``: Stop immediately after idle time expires, regardless of running
+jobs or SSH connections."""
+
+    @classmethod
+    def from_str(cls, mode: str) -> 'AutostopWaitFor':
+        """Returns the enum value for the given string."""
+        if mode.lower() == cls.JOBS.value:
+            return cls.JOBS
+        elif mode.lower() == cls.JOBS_AND_SSH.value:
+            return cls.JOBS_AND_SSH
+        elif mode.lower() == cls.NONE.value:
+            return cls.NONE
+        else:
+            with ux_utils.print_exception_no_traceback():
+                raise ValueError(f'Unsupported autostop wait mode: '
+                                 f'{mode}. The mode must be either '
+                                 f'\'{cls.JOBS_AND_SSH.value}\', '
+                                 f'\'{cls.JOBS.value}\', or '
+                                 f'\'{cls.NONE.value}\'. ')
+
+
+DEFAULT_AUTOSTOP_WAIT_FOR: AutostopWaitFor = AutostopWaitFor.JOBS_AND_SSH
+
+
 class AutostopConfig:
     """Autostop configuration."""
 
@@ -37,12 +89,14 @@ class AutostopConfig:
                  autostop_idle_minutes: int,
                  boot_time: float,
                  backend: Optional[str],
+                 wait_for: AutostopWaitFor,
                  down: bool = False):
         assert autostop_idle_minutes < 0 or backend is not None, (
             autostop_idle_minutes, backend)
         self.autostop_idle_minutes = autostop_idle_minutes
         self.boot_time = boot_time
         self.backend = backend
+        self.wait_for = wait_for
         self.down = down
 
     def __setstate__(self, state: dict):
@@ -53,15 +107,18 @@ class AutostopConfig:
 def get_autostop_config() -> AutostopConfig:
     config_str = configs.get_config(_AUTOSTOP_CONFIG_KEY)
     if config_str is None:
-        return AutostopConfig(-1, -1, None)
+        return AutostopConfig(-1, -1, None, DEFAULT_AUTOSTOP_WAIT_FOR)
     return pickle.loads(config_str)
 
 
-def set_autostop(idle_minutes: int, backend: Optional[str], down: bool) -> None:
+def set_autostop(idle_minutes: int, backend: Optional[str],
+                 wait_for: AutostopWaitFor, down: bool) -> None:
     boot_time = psutil.boot_time()
-    autostop_config = AutostopConfig(idle_minutes, boot_time, backend, down)
+    autostop_config = AutostopConfig(idle_minutes, boot_time, backend, wait_for,
+                                     down)
     configs.set_config(_AUTOSTOP_CONFIG_KEY, pickle.dumps(autostop_config))
-    logger.debug(f'set_autostop(): idle_minutes {idle_minutes}, down {down}.')
+    logger.debug(f'set_autostop(): idle_minutes {idle_minutes}, down {down}, '
+                 f'wait_for {wait_for.value}.')
     # Reset timer whenever an autostop setting is submitted, i.e. the idle
     # time will be counted from now.
     set_last_active_time_to_now()
@@ -107,6 +164,28 @@ def set_last_active_time_to_now() -> None:
     configs.set_config(_AUTOSTOP_LAST_ACTIVE_TIME, str(time.time()))
 
 
+def has_active_ssh_sessions() -> bool:
+    """Returns True if there are any active SSH sessions on the node."""
+    try:
+        # /dev/pts is a virtual filesystem that contains the pseudo-terminal
+        # devices. ptmx is the pseudo-terminal multiplexer, which is the
+        # "master" device that creates new pseudo-terminal devices, so we
+        # exclude it from the count.
+        proc = subprocess.run('ls /dev/pts | grep -v ptmx | wc -l',
+                              capture_output=True,
+                              text=True,
+                              check=False,
+                              shell=True)
+        if proc.returncode != 0:
+            logger.warning(f'SSH session check command failed with return code '
+                           f'{proc.returncode}.')
+            return False
+        return int(proc.stdout.strip()) > 0
+    except Exception as e:  # pylint: disable=broad-except
+        logger.warning(f'Error checking active SSH sessions: {e}.')
+        return False
+
+
 class AutostopCodeGen:
     """Code generator for autostop utility functions.
 
@@ -114,13 +193,22 @@ class AutostopCodeGen:
 
       >> codegen = AutostopCodeGen.set_autostop(...)
     """
-    _PREFIX = ['from sky.skylet import autostop_lib']
+    _PREFIX = ['from sky.skylet import autostop_lib, constants']
 
     @classmethod
-    def set_autostop(cls, idle_minutes: int, backend: str, down: bool) -> str:
+    def set_autostop(cls,
+                     idle_minutes: int,
+                     backend: str,
+                     wait_for: Optional[AutostopWaitFor],
+                     down: bool = False) -> str:
+        if wait_for is None:
+            wait_for = DEFAULT_AUTOSTOP_WAIT_FOR
         code = [
-            f'autostop_lib.set_autostop({idle_minutes}, {backend!r},'
-            f' {down})',
+            f'\nif getattr(constants, "SKYLET_LIB_VERSION", 1) < 4: '
+            f'\n autostop_lib.set_autostop({idle_minutes}, {backend!r}, {down})'
+            f'\nelse: '
+            f'\n autostop_lib.set_autostop({idle_minutes}, {backend!r}, '
+            f'autostop_lib.{wait_for}, {down})',
         ]
         return cls._build(code)
 

sky/skylet/constants.py

@@ -90,11 +90,11 @@ TASK_ID_LIST_ENV_VAR = f'{SKYPILOT_ENV_VAR_PREFIX}TASK_IDS'
 # cluster yaml is updated.
 #
 # TODO(zongheng,zhanghao): make the upgrading of skylet automatic?
-SKYLET_VERSION = '15'
+SKYLET_VERSION = '16'
 # The version of the lib files that skylet/jobs use. Whenever there is an API
 # change for the job_lib or log_lib, we need to bump this version, so that the
 # user can be notified to update their SkyPilot version on the remote cluster.
-SKYLET_LIB_VERSION = 3
+SKYLET_LIB_VERSION = 4
 SKYLET_VERSION_FILE = '~/.sky/skylet_version'
 
 # Docker default options
@@ -369,6 +369,7 @@ OVERRIDEABLE_CONFIG_KEYS_IN_TASK: List[Tuple[str, ...]] = [
     ('docker', 'run_options'),
     ('nvidia_gpus', 'disable_ecc'),
     ('ssh', 'pod_config'),
+    ('kubernetes', 'custom_metadata'),
     ('kubernetes', 'pod_config'),
     ('kubernetes', 'provision_timeout'),
     ('kubernetes', 'dws'),

sky/skylet/events.py

@@ -137,23 +137,37 @@ class AutostopEvent(SkyletEvent):
             logger.debug('autostop_config not set. Skipped.')
             return
 
-        if (job_lib.is_cluster_idle() and
-                not managed_job_state.get_num_alive_jobs()):
-            idle_minutes = (time.time() -
-                            autostop_lib.get_last_active_time()) // 60
+        ignore_idle_check = (
+            autostop_config.wait_for == autostop_lib.AutostopWaitFor.NONE)
+        is_idle = True
+        if not ignore_idle_check:
+            if not job_lib.is_cluster_idle(
+            ) or managed_job_state.get_num_alive_jobs() or (
+                    autostop_config.wait_for
+                    == autostop_lib.AutostopWaitFor.JOBS_AND_SSH and
+                    autostop_lib.has_active_ssh_sessions()):
+                is_idle = False
+
+        if ignore_idle_check or is_idle:
+            minutes_since_last_active = (
+                time.time() - autostop_lib.get_last_active_time()) // 60
             logger.debug(
-                f'Idle minutes: {idle_minutes}, '
-                f'AutoStop config: {autostop_config.autostop_idle_minutes}')
+                f'Minutes since last active: {minutes_since_last_active}, '
+                f'AutoStop idle minutes: '
+                f'{autostop_config.autostop_idle_minutes}, '
+                f'Wait for: {autostop_config.wait_for.value}')
         else:
             autostop_lib.set_last_active_time_to_now()
-            idle_minutes = -1
-            logger.debug(
-                'Not idle. Reset idle minutes.'
-                f'AutoStop config: {autostop_config.autostop_idle_minutes}')
-        if idle_minutes >= autostop_config.autostop_idle_minutes:
+            minutes_since_last_active = -1
+            logger.debug('Not idle. Reset idle minutes. '
+                         f'AutoStop idle minutes: '
+                         f'{autostop_config.autostop_idle_minutes}, '
+                         f'Wait for: {autostop_config.wait_for.value}')
+        if minutes_since_last_active >= autostop_config.autostop_idle_minutes:
             logger.info(
-                f'{idle_minutes} idle minutes reached; threshold: '
-                f'{autostop_config.autostop_idle_minutes} minutes. Stopping.')
+                f'{minutes_since_last_active} minute(s) since last active; '
+                f'threshold: {autostop_config.autostop_idle_minutes} minutes. '
+                f'Stopping.')
             self._stop_cluster(autostop_config)
 
     def _stop_cluster(self, autostop_config):

sky/templates/kubernetes-loadbalancer.yml.j2

@@ -12,6 +12,8 @@ service_spec:
       {%- for key, value in annotations.items() %}
       {{ key }}: {{ value|tojson }}
       {%- endfor %}
+      {# Note: It's ok to add cloud-specific annotations here since they will be ignored by other clouds #}
+      service.beta.kubernetes.io/coreweave-load-balancer-type: public
   spec:
     type: LoadBalancer
     selector:

sky/utils/schemas.py

@@ -6,6 +6,7 @@ https://json-schema.org/
 import enum
 from typing import Any, Dict, List, Tuple
 
+from sky.skylet import autostop_lib
 from sky.skylet import constants
 from sky.utils import kubernetes_enums
 
@@ -65,6 +66,11 @@ _AUTOSTOP_SCHEMA = {
                 'down': {
                     'type': 'boolean',
                 },
+                'wait_for': {
+                    'type': 'string',
+                    'case_insensitive_enum':
+                        autostop_lib.AutostopWaitFor.supported_modes(),
+                }
             },
         },
     ],

{skypilot_nightly-1.0.0.dev20250801.dist-info → skypilot_nightly-1.0.0.dev20250804.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skypilot-nightly
-Version: 1.0.0.dev20250801
+Version: 1.0.0.dev20250804
 Summary: SkyPilot: Run AI on Any Infra — Unified, Faster, Cheaper.
 Author: SkyPilot Team
 License: Apache 2.0
@@ -121,7 +121,7 @@ Requires-Dist: vastai-sdk>=0.1.12; extra == "vast"
 Provides-Extra: vsphere
 Requires-Dist: pyvmomi==8.0.1.0.2; extra == "vsphere"
 Provides-Extra: nebius
-Requires-Dist: nebius>=0.2.37; extra == "nebius"
+Requires-Dist: nebius>=0.2.47; extra == "nebius"
 Requires-Dist: awscli>=1.27.10; extra == "nebius"
 Requires-Dist: botocore>=1.29.10; extra == "nebius"
 Requires-Dist: boto3>=1.26.1; extra == "nebius"
@@ -177,7 +177,7 @@ Requires-Dist: azure-core>=1.24.0; extra == "all"
 Requires-Dist: azure-common; extra == "all"
 Requires-Dist: vastai-sdk>=0.1.12; extra == "all"
 Requires-Dist: pyvmomi==8.0.1.0.2; extra == "all"
-Requires-Dist: nebius>=0.2.37; extra == "all"
+Requires-Dist: nebius>=0.2.47; extra == "all"
 Requires-Dist: awscli>=1.27.10; extra == "all"
 Requires-Dist: botocore>=1.29.10; extra == "all"
 Requires-Dist: boto3>=1.26.1; extra == "all"
@@ -235,6 +235,7 @@ Dynamic: summary
 ----
 
 :fire: *News* :fire:
+- [Jul 2025] Run distributed **RL training for LLMs** with Verl (PPO, GRPO) on any cloud: [**example**](./llm/verl/)
 - [Jul 2025] 🎉 SkyPilot v0.10.0 released! [**blog post**](https://blog.skypilot.co/announcing-skypilot-0.10.0/), [**release notes**](https://github.com/skypilot-org/skypilot/releases/tag/v0.10.0)
 - [Jul 2025] Finetune **Llama4** on any distributed cluster/cloud: [**example**](./llm/llama-4-finetuning/)
 - [Jul 2025] Two-part blog series, `The Evolution of AI Job Orchestration`: (1) [Running AI jobs on GPU Neoclouds](https://blog.skypilot.co/ai-job-orchestration-pt1-gpu-neoclouds/), (2) [The AI-Native Control Plane & Orchestration that Finally Works for ML](https://blog.skypilot.co/ai-job-orchestration-pt2-ai-control-plane/)