skypilot-nightly 1.0.0.dev20250731__py3-none-any.whl → 1.0.0.dev20250802__py3-none-any.whl

sky/jobs/client/sdk.py

@@ -2,11 +2,11 @@
 import json
 import typing
 from typing import Dict, List, Optional, Union
-import webbrowser
 
 import click
 
 from sky import sky_logging
+from sky.adaptors import common as adaptors_common
 from sky.client import common as client_common
 from sky.client import sdk
 from sky.serve.client import impl
@@ -23,9 +23,13 @@ from sky.utils import dag_utils
 
 if typing.TYPE_CHECKING:
     import io
+    import webbrowser
 
     import sky
     from sky.serve import serve_utils
+else:
+    # only used in dashboard()
+    webbrowser = adaptors_common.LazyImport('webbrowser')
 
 logger = sky_logging.init_logger(__name__)
 

sky/provision/runpod/utils.py

@@ -270,18 +270,17 @@ def launch(cluster_name: str, node_type: str, instance_type: str, region: str,
            docker_login_config: Optional[Dict[str, str]]) -> str:
     """Launches an instance with the given parameters.
 
-    Converts the instance_type to the RunPod GPU name, finds the specs for the
-    GPU, and launches the instance.
+    For CPU instances, we directly use the instance_type for launching the
+    instance.
+
+    For GPU instances, we convert the instance_type to the RunPod GPU name,
+    and finds the specs for the GPU, before launching the instance.
 
     Returns:
         instance_id: The instance ID.
     """
     name = f'{cluster_name}-{node_type}'
-    gpu_type = GPU_NAME_MAP[instance_type.split('_')[1]]
-    gpu_quantity = int(instance_type.split('_')[0].replace('x', ''))
-    cloud_type = instance_type.split('_')[2]
 
-    gpu_specs = runpod.runpod.get_gpu(gpu_type)
     # TODO(zhwu): keep this align with setups in
     # `provision.kuberunetes.instance.py`
     setup_cmd = (
@@ -329,12 +328,7 @@ def launch(cluster_name: str, node_type: str, instance_type: str, region: str,
     params = {
         'name': name,
         'image_name': image_name_formatted,
-        'gpu_type_id': gpu_type,
-        'cloud_type': cloud_type,
         'container_disk_in_gb': disk_size,
-        'min_vcpu_count': 4 * gpu_quantity,
-        'min_memory_in_gb': gpu_specs['memoryInGb'] * gpu_quantity,
-        'gpu_count': gpu_quantity,
         'country_code': region,
         'data_center_id': zone,
         'ports': ports_str,
@@ -343,12 +337,33 @@ def launch(cluster_name: str, node_type: str, instance_type: str, region: str,
         'template_id': template_id,
     }
 
+    # GPU instance types start with f'{gpu_count}x',
+    # CPU instance types start with 'cpu'.
+    is_cpu_instance = instance_type.startswith('cpu')
+    if is_cpu_instance:
+        # RunPod CPU instances can be uniquely identified by the instance_id.
+        params.update({
+            'instance_id': instance_type,
+        })
+    else:
+        gpu_type = GPU_NAME_MAP[instance_type.split('_')[1]]
+        gpu_quantity = int(instance_type.split('_')[0].replace('x', ''))
+        cloud_type = instance_type.split('_')[2]
+        gpu_specs = runpod.runpod.get_gpu(gpu_type)
+        params.update({
+            'gpu_type_id': gpu_type,
+            'cloud_type': cloud_type,
+            'min_vcpu_count': 4 * gpu_quantity,
+            'min_memory_in_gb': gpu_specs['memoryInGb'] * gpu_quantity,
+            'gpu_count': gpu_quantity,
+        })
+
     if preemptible is None or not preemptible:
         new_instance = runpod.runpod.create_pod(**params)
     else:
         new_instance = runpod_commands.create_spot_pod(
             bid_per_gpu=bid_per_gpu,
-            **params,
+            **params,  # type: ignore[arg-type]
         )
 
     return new_instance['id']

sky/resources.py

@@ -20,6 +20,7 @@ from sky.provision import docker_utils
 from sky.provision.gcp import constants as gcp_constants
 from sky.provision.kubernetes import utils as kubernetes_utils
 from sky.provision.nebius import constants as nebius_constants
+from sky.skylet import autostop_lib
 from sky.skylet import constants
 from sky.utils import accelerator_registry
 from sky.utils import annotations
@@ -69,14 +70,18 @@ class AutostopConfig:
     # flags.
     idle_minutes: int = 0
     down: bool = False
+    wait_for: Optional[autostop_lib.AutostopWaitFor] = None
 
     def to_yaml_config(self) -> Union[Literal[False], Dict[str, Any]]:
         if not self.enabled:
             return False
-        return {
+        config: Dict[str, Any] = {
             'idle_minutes': self.idle_minutes,
             'down': self.down,
         }
+        if self.wait_for is not None:
+            config['wait_for'] = self.wait_for.value
+        return config
 
     @classmethod
     def from_yaml_config(
@@ -104,6 +109,9 @@ class AutostopConfig:
                 autostop_config.idle_minutes = config['idle_minutes']
             if 'down' in config:
                 autostop_config.down = config['down']
+            if 'wait_for' in config:
+                autostop_config.wait_for = (
+                    autostop_lib.AutostopWaitFor.from_str(config['wait_for']))
             return autostop_config
 
         return None
@@ -958,15 +966,18 @@ class Resources:
             valid_volumes.append(volume)
         self._volumes = valid_volumes
 
-    def override_autostop_config(self,
-                                 down: bool = False,
-                                 idle_minutes: Optional[int] = None) -> None:
+    def override_autostop_config(
+            self,
+            down: bool = False,
+            idle_minutes: Optional[int] = None,
+            wait_for: Optional[autostop_lib.AutostopWaitFor] = None) -> None:
         """Override autostop config to the resource.
 
         Args:
             down: If true, override the autostop config to use autodown.
             idle_minutes: If not None, override the idle minutes to autostop or
                 autodown.
+            wait_for: If not None, override the wait mode.
         """
         if not down and idle_minutes is None:
             return
@@ -976,6 +987,8 @@ class Resources:
             self._autostop_config.down = down
         if idle_minutes is not None:
             self._autostop_config.idle_minutes = idle_minutes
+        if wait_for is not None:
+            self._autostop_config.wait_for = wait_for
 
     def is_launchable(self) -> bool:
         """Returns whether the resource is launchable."""

sky/server/constants.py

@@ -10,7 +10,7 @@ from sky.skylet import constants
 # based on version info is needed.
 # For more details and code guidelines, refer to:
 # https://docs.skypilot.co/en/latest/developers/CONTRIBUTING.html#backward-compatibility-guidelines
-API_VERSION = 12
+API_VERSION = 13
 
 # The minimum peer API version that the code should still work with.
 # Notes (dev):

sky/server/daemons.py

@@ -0,0 +1,164 @@
+"""Internal server daemons that run in the background."""
+import dataclasses
+import os
+import time
+from typing import Callable
+
+from sky import sky_logging
+from sky import skypilot_config
+from sky.server import constants as server_constants
+from sky.utils import common
+from sky.utils import env_options
+from sky.utils import ux_utils
+
+logger = sky_logging.init_logger(__name__)
+
+
+@dataclasses.dataclass
+class InternalRequestDaemon:
+    """Internal daemon that runs an event in the background."""
+
+    id: str
+    name: str
+    event_fn: Callable[[], None]
+    default_log_level: str = 'INFO'
+
+    def refresh_log_level(self) -> int:
+        # pylint: disable=import-outside-toplevel
+        import logging
+
+        try:
+            # Refresh config within the while loop.
+            # Since this is a long running daemon,
+            # reload_config_for_new_request()
+            # is not called in between the event runs.
+            skypilot_config.safe_reload_config()
+            # Get the configured log level for the daemon inside the event loop
+            # in case the log level changes after the API server is started.
+            level_str = skypilot_config.get_nested(
+                ('daemons', self.id, 'log_level'), self.default_log_level)
+            return getattr(logging, level_str.upper())
+        except AttributeError:
+            # Bad level should be rejected by
+            # schema validation, just in case.
+            logger.warning(f'Invalid log level: {level_str}, using DEBUG')
+            return logging.DEBUG
+        except Exception as e:  # pylint: disable=broad-except
+            logger.exception(f'Error refreshing log level for {self.id}: {e}')
+            return logging.DEBUG
+
+    def run_event(self):
+        """Run the event."""
+
+        # Disable logging for periodic refresh to avoid the usage message being
+        # sent multiple times.
+        os.environ[env_options.Options.DISABLE_LOGGING.env_key] = '1'
+
+        level = self.refresh_log_level()
+        while True:
+            try:
+                with ux_utils.enable_traceback(), \
+                    sky_logging.set_sky_logging_levels(level):
+                    sky_logging.reload_logger()
+                    level = self.refresh_log_level()
+                    self.event_fn()
+            except Exception:  # pylint: disable=broad-except
+                # It is OK to fail to run the event, as the event is not
+                # critical, but we should log the error.
+                logger.exception(
+                    f'Error running {self.name} event. '
+                    f'Restarting in '
+                    f'{server_constants.DAEMON_RESTART_INTERVAL_SECONDS} '
+                    'seconds...')
+                time.sleep(server_constants.DAEMON_RESTART_INTERVAL_SECONDS)
+
+
+def refresh_cluster_status_event():
+    """Periodically refresh the cluster status."""
+    # pylint: disable=import-outside-toplevel
+    from sky import core
+
+    logger.info('=== Refreshing cluster status ===')
+    # This periodically refresh will hold the lock for the cluster being
+    # refreshed, but it is OK because other operations will just wait for
+    # the lock and get the just refreshed status without refreshing again.
+    core.status(refresh=common.StatusRefreshMode.FORCE, all_users=True)
+    logger.info('Status refreshed. Sleeping '
+                f'{server_constants.CLUSTER_REFRESH_DAEMON_INTERVAL_SECONDS}'
+                ' seconds for the next refresh...\n')
+    time.sleep(server_constants.CLUSTER_REFRESH_DAEMON_INTERVAL_SECONDS)
+
+
+def refresh_volume_status_event():
+    """Periodically refresh the volume status."""
+    # pylint: disable=import-outside-toplevel
+    from sky.volumes.server import core
+
+    # Disable logging for periodic refresh to avoid the usage message being
+    # sent multiple times.
+    os.environ[env_options.Options.DISABLE_LOGGING.env_key] = '1'
+
+    logger.info('=== Refreshing volume status ===')
+    core.volume_refresh()
+    logger.info('Volume status refreshed. Sleeping '
+                f'{server_constants.VOLUME_REFRESH_DAEMON_INTERVAL_SECONDS}'
+                ' seconds for the next refresh...\n')
+    time.sleep(server_constants.VOLUME_REFRESH_DAEMON_INTERVAL_SECONDS)
+
+
+def managed_job_status_refresh_event():
+    """Refresh the managed job status for controller consolidation mode."""
+    # pylint: disable=import-outside-toplevel
+    from sky.jobs import utils as managed_job_utils
+    if not managed_job_utils.is_consolidation_mode():
+        return
+    # We run the recovery logic before starting the event loop as those two are
+    # conflicting. Check PERSISTENT_RUN_RESTARTING_SIGNAL_FILE for details.
+    from sky.utils import controller_utils
+    if controller_utils.high_availability_specified(
+            controller_utils.Controllers.JOBS_CONTROLLER.value.cluster_name):
+        managed_job_utils.ha_recovery_for_consolidation_mode()
+    # After recovery, we start the event loop.
+    from sky.skylet import events
+    refresh_event = events.ManagedJobEvent()
+    scheduling_event = events.ManagedJobSchedulingEvent()
+    logger.info('=== Running managed job event ===')
+    refresh_event.run()
+    scheduling_event.run()
+    time.sleep(events.EVENT_CHECKING_INTERVAL_SECONDS)
+
+
+def sky_serve_status_refresh_event():
+    """Refresh the sky serve status for controller consolidation mode."""
+    # pylint: disable=import-outside-toplevel
+    from sky.serve import serve_utils
+    if not serve_utils.is_consolidation_mode():
+        return
+    # TODO(tian): Add HA recovery logic.
+    from sky.skylet import events
+    event = events.ServiceUpdateEvent()
+    logger.info('=== Running serve status refresh event ===')
+    event.run()
+    time.sleep(events.EVENT_CHECKING_INTERVAL_SECONDS)
+
+
+# Register the events to run in the background.
+INTERNAL_REQUEST_DAEMONS = [
+    # This status refresh daemon can cause the autostopp'ed/autodown'ed cluster
+    # set to updated status automatically, without showing users the hint of
+    # cluster being stopped or down when `sky status -r` is called.
+    InternalRequestDaemon(id='skypilot-status-refresh-daemon',
+                          name='status',
+                          event_fn=refresh_cluster_status_event,
+                          default_log_level='DEBUG'),
+    # Volume status refresh daemon to update the volume status periodically.
+    InternalRequestDaemon(id='skypilot-volume-status-refresh-daemon',
+                          name='volume',
+                          event_fn=refresh_volume_status_event),
+    InternalRequestDaemon(id='managed-job-status-refresh-daemon',
+                          name='managed-job-status',
+                          event_fn=managed_job_status_refresh_event),
+    InternalRequestDaemon(id='sky-serve-status-refresh-daemon',
+                          name='sky-serve-status',
+                          event_fn=sky_serve_status_refresh_event),
+]

sky/server/requests/payloads.py

@@ -33,6 +33,7 @@ from sky import sky_logging
 from sky import skypilot_config
 from sky.adaptors import common as adaptors_common
 from sky.server import common
+from sky.skylet import autostop_lib
 from sky.skylet import constants
 from sky.usage import constants as usage_constants
 from sky.usage import usage_lib
@@ -312,6 +313,7 @@ class StartBody(RequestBody):
     """The request body for the start endpoint."""
     cluster_name: str
     idle_minutes_to_autostop: Optional[int] = None
+    wait_for: Optional[autostop_lib.AutostopWaitFor] = None
     retry_until_up: bool = False
     down: bool = False
     force: bool = False
@@ -321,6 +323,7 @@ class AutostopBody(RequestBody):
     """The request body for the autostop endpoint."""
     cluster_name: str
     idle_minutes: int
+    wait_for: Optional[autostop_lib.AutostopWaitFor] = None
     down: bool = False
 
 

sky/server/requests/requests.py

@@ -24,12 +24,11 @@ from sky import sky_logging
 from sky import skypilot_config
 from sky.server import common as server_common
 from sky.server import constants as server_constants
+from sky.server import daemons
 from sky.server.requests import payloads
 from sky.server.requests.serializers import decoders
 from sky.server.requests.serializers import encoders
-from sky.utils import common
 from sky.utils import common_utils
-from sky.utils import env_options
 from sky.utils import subprocess_utils
 from sky.utils import ux_utils
 from sky.utils.db import db_utils
@@ -307,127 +306,6 @@ def kill_cluster_requests(cluster_name: str, exclude_request_name: str):
     kill_requests(request_ids)
 
 
-def refresh_cluster_status_event():
-    """Periodically refresh the cluster status."""
-    # pylint: disable=import-outside-toplevel
-    from sky import core
-
-    # Disable logging for periodic refresh to avoid the usage message being
-    # sent multiple times.
-    os.environ[env_options.Options.DISABLE_LOGGING.env_key] = '1'
-
-    while True:
-        logger.info('=== Refreshing cluster status ===')
-        # This periodically refresh will hold the lock for the cluster being
-        # refreshed, but it is OK because other operations will just wait for
-        # the lock and get the just refreshed status without refreshing again.
-        core.status(refresh=common.StatusRefreshMode.FORCE, all_users=True)
-        logger.info(
-            'Status refreshed. Sleeping '
-            f'{server_constants.CLUSTER_REFRESH_DAEMON_INTERVAL_SECONDS}'
-            ' seconds for the next refresh...\n')
-        time.sleep(server_constants.CLUSTER_REFRESH_DAEMON_INTERVAL_SECONDS)
-
-
-def refresh_volume_status_event():
-    """Periodically refresh the volume status."""
-    # pylint: disable=import-outside-toplevel
-    from sky.volumes.server import core
-
-    # Disable logging for periodic refresh to avoid the usage message being
-    # sent multiple times.
-    os.environ[env_options.Options.DISABLE_LOGGING.env_key] = '1'
-
-    while True:
-        logger.info('=== Refreshing volume status ===')
-        core.volume_refresh()
-        logger.info('Volume status refreshed. Sleeping '
-                    f'{server_constants.VOLUME_REFRESH_DAEMON_INTERVAL_SECONDS}'
-                    ' seconds for the next refresh...\n')
-        time.sleep(server_constants.VOLUME_REFRESH_DAEMON_INTERVAL_SECONDS)
-
-
-def managed_job_status_refresh_event():
-    """Refresh the managed job status for controller consolidation mode."""
-    # pylint: disable=import-outside-toplevel
-    from sky.jobs import utils as managed_job_utils
-    if not managed_job_utils.is_consolidation_mode():
-        return
-    # We run the recovery logic before starting the event loop as those two are
-    # conflicting. Check PERSISTENT_RUN_RESTARTING_SIGNAL_FILE for details.
-    from sky.utils import controller_utils
-    if controller_utils.high_availability_specified(
-            controller_utils.Controllers.JOBS_CONTROLLER.value.cluster_name):
-        managed_job_utils.ha_recovery_for_consolidation_mode()
-    # After recovery, we start the event loop.
-    from sky.skylet import events
-    refresh_event = events.ManagedJobEvent()
-    scheduling_event = events.ManagedJobSchedulingEvent()
-    while True:
-        logger.info('=== Running managed job event ===')
-        refresh_event.run()
-        scheduling_event.run()
-        time.sleep(events.EVENT_CHECKING_INTERVAL_SECONDS)
-
-
-def sky_serve_status_refresh_event():
-    """Refresh the managed job status for controller consolidation mode."""
-    # pylint: disable=import-outside-toplevel
-    from sky.serve import serve_utils
-    if not serve_utils.is_consolidation_mode():
-        return
-    # TODO(tian): Add HA recovery logic.
-    from sky.skylet import events
-    event = events.ServiceUpdateEvent()
-    while True:
-        time.sleep(events.EVENT_CHECKING_INTERVAL_SECONDS)
-        event.run()
-
-
-@dataclasses.dataclass
-class InternalRequestDaemon:
-    """Internal daemon that runs an event in the background."""
-
-    id: str
-    name: str
-    event_fn: Callable[[], None]
-
-    def run_event(self):
-        """Run the event."""
-        while True:
-            with ux_utils.enable_traceback():
-                try:
-                    self.event_fn()
-                    break
-                except Exception:  # pylint: disable=broad-except
-                    # It is OK to fail to run the event, as the event is not
-                    # critical, but we should log the error.
-                    logger.exception(
-                        f'Error running {self.name} event. '
-                        f'Restarting in '
-                        f'{server_constants.DAEMON_RESTART_INTERVAL_SECONDS} '
-                        'seconds...')
-                    time.sleep(server_constants.DAEMON_RESTART_INTERVAL_SECONDS)
-
-
-# Register the events to run in the background.
-INTERNAL_REQUEST_DAEMONS = [
-    # This status refresh daemon can cause the autostopp'ed/autodown'ed cluster
-    # set to updated status automatically, without showing users the hint of
-    # cluster being stopped or down when `sky status -r` is called.
-    InternalRequestDaemon(id='skypilot-status-refresh-daemon',
-                          name='status',
-                          event_fn=refresh_cluster_status_event),
-    # Volume status refresh daemon to update the volume status periodically.
-    InternalRequestDaemon(id='skypilot-volume-status-refresh-daemon',
-                          name='volume',
-                          event_fn=refresh_volume_status_event),
-    InternalRequestDaemon(id='managed-job-status-refresh-daemon',
-                          name='managed-job-status',
-                          event_fn=managed_job_status_refresh_event),
-]
-
-
 def kill_requests(request_ids: Optional[List[str]] = None,
                   user_id: Optional[str] = None) -> List[str]:
     """Kill a SkyPilot API request and set its status to cancelled.
@@ -458,7 +336,7 @@ def kill_requests(request_ids: Optional[List[str]] = None,
             # Skip internal requests. The internal requests are scheduled with
             # request_id in range(len(INTERNAL_REQUEST_EVENTS)).
             if request_record.request_id in set(
-                    event.id for event in INTERNAL_REQUEST_DAEMONS):
+                    event.id for event in daemons.INTERNAL_REQUEST_DAEMONS):
                 continue
             if request_record.status > RequestStatus.RUNNING:
                 logger.debug(f'Request {request_id} already finished')

sky/server/server.py

@@ -46,6 +46,7 @@ from sky.serve.server import server as serve_rest
 from sky.server import common
 from sky.server import config as server_config
 from sky.server import constants as server_constants
+from sky.server import daemons
 from sky.server import metrics
 from sky.server import state
 from sky.server import stream_utils
@@ -482,7 +483,7 @@ async def lifespan(app: fastapi.FastAPI):  # pylint: disable=redefined-outer-nam
     """FastAPI lifespan context manager."""
     del app  # unused
     # Startup: Run background tasks
-    for event in requests_lib.INTERNAL_REQUEST_DAEMONS:
+    for event in daemons.INTERNAL_REQUEST_DAEMONS:
         try:
             executor.schedule_request(
                 request_id=event.id,

sky/server/uvicorn.py

@@ -16,6 +16,7 @@ import uvicorn
 from uvicorn.supervisors import multiprocess
 
 from sky import sky_logging
+from sky.server import daemons
 from sky.server import state
 from sky.server.requests import requests as requests_lib
 from sky.skylet import constants
@@ -120,7 +121,7 @@ class Server(uvicorn.Server):
             # Proactively cancel internal requests and logs requests since
             # they can run for infinite time.
             internal_request_ids = [
-                d.id for d in requests_lib.INTERNAL_REQUEST_DAEMONS
+                d.id for d in daemons.INTERNAL_REQUEST_DAEMONS
             ]
             if time.time() - start_time > _WAIT_REQUESTS_TIMEOUT_SECONDS:
                 logger.warning('Timeout waiting for on-going requests to '

sky/setup_files/dependencies.py

@@ -177,7 +177,7 @@ extras_require: Dict[str, List[str]] = {
         # 'vsphere-automation-sdk @ git+https://github.com/vmware/vsphere-automation-sdk-python.git@v8.0.1.0' pylint: disable=line-too-long
     ],
     'nebius': [
-        'nebius>=0.2.37',
+        'nebius>=0.2.47',
     ] + aws_dependencies,
     'hyperbolic': [],  # No dependencies needed for hyperbolic
     'server': server_dependencies,

sky/sky_logging.py

@@ -171,6 +171,36 @@ def set_logging_level(logger: str, level: int):
         logger.setLevel(original_level)
 
 
+@contextlib.contextmanager
+def set_sky_logging_levels(level: int):
+    """Set the logging level for all loggers."""
+    # Turn off logger
+    previous_levels = {}
+    for logger_name in logging.Logger.manager.loggerDict:
+        if logger_name.startswith('sky'):
+            logger = logging.getLogger(logger_name)
+            previous_levels[logger_name] = logger.level
+            logger.setLevel(level)
+    if level == logging.DEBUG:
+        previous_show_debug_info = env_options.Options.SHOW_DEBUG_INFO.get()
+        os.environ[env_options.Options.SHOW_DEBUG_INFO.env_key] = '1'
+    try:
+        yield
+    finally:
+        # Restore logger
+        for logger_name in logging.Logger.manager.loggerDict:
+            if logger_name.startswith('sky'):
+                logger = logging.getLogger(logger_name)
+                try:
+                    logger.setLevel(previous_levels[logger_name])
+                except KeyError:
+                    # New loggers maybe initialized after the context manager,
+                    # no need to restore the level.
+                    pass
+        if level == logging.DEBUG and not previous_show_debug_info:
+            os.environ.pop(env_options.Options.SHOW_DEBUG_INFO.env_key)
+
+
 def logging_enabled(logger: logging.Logger, level: int) -> bool:
     return logger.level <= level