skypilot-nightly 1.0.0.dev20250827__py3-none-any.whl → 1.0.0.dev20250828__py3-none-any.whl

sky/provision/kubernetes/instance.py

@@ -1,5 +1,6 @@
 """Kubernetes instance provisioning."""
 import copy
+import datetime
 import json
 import re
 import time
@@ -1254,9 +1255,11 @@ def get_cluster_info(
         provider_config=provider_config)
 
 
-def _get_pod_termination_reason(pod: Any) -> str:
+def _get_pod_termination_reason(pod: Any, cluster_name: str) -> str:
+    """Get pod termination reason and write to cluster events."""
     reasons = []
-    if pod.status.container_statuses:
+    latest_timestamp = pod.status.start_time or datetime.datetime.min
+    if pod.status and pod.status.container_statuses:
         for container_status in pod.status.container_statuses:
             terminated = container_status.state.terminated
             if terminated:
@@ -1264,20 +1267,38 @@ def _get_pod_termination_reason(pod: Any) -> str:
                 reason = terminated.reason
                 if exit_code == 0:
                     # skip exit 0 (non-failed) just for sanity
+                    logger.debug(f'{pod.metadata.name}/{container_status.name} '
+                                 'had exit code 0. Skipping.')
                     continue
                 if reason is None:
                     # just in-case reason is None, have default for debugging
                     reason = f'exit({exit_code})'
                 reasons.append(reason)
+                if terminated.finished_at > latest_timestamp:
+                    latest_timestamp = terminated.finished_at
+
             # TODO (kyuds): later, if needed, query `last_state` too.
 
+    if not reasons:
+        return ''
+
     # Normally we will have a single container per pod for skypilot
     # but doing this just in-case there are multiple containers.
-    return ' | '.join(reasons)
+    pod_reason = ' | '.join(reasons)
+
+    global_user_state.add_cluster_event(
+        cluster_name,
+        None,
+        f'[kubernetes pod {pod.metadata.name} terminated] {pod_reason}',
+        global_user_state.ClusterEventType.DEBUG,
+        transitioned_at=int(latest_timestamp.timestamp()),
+    )
+    return pod_reason
 
 
 def _get_pod_missing_reason(context: Optional[str], namespace: str,
                             cluster_name: str, pod_name: str) -> Optional[str]:
+    """Get events for missing pod and write to cluster events."""
     logger.debug(f'Analyzing events for pod {pod_name}')
     pod_field_selector = (
         f'involvedObject.kind=Pod,involvedObject.name={pod_name}')
@@ -1293,6 +1314,8 @@ def _get_pod_missing_reason(context: Optional[str], namespace: str,
     last_scheduled_node = None
     insert_new_pod_event = True
     new_event_inserted = False
+    inserted_pod_events = 0
+
     for event in pod_events:
         if event.reason == 'Scheduled':
             pattern = r'Successfully assigned (\S+) to (\S+)'
@@ -1313,10 +1336,18 @@ def _get_pod_missing_reason(context: Optional[str], namespace: str,
                     transitioned_at=int(
                         event.metadata.creation_timestamp.timestamp()),
                     expose_duplicate_error=True)
+                logger.debug(f'[pod {pod_name}] encountered new pod event: '
+                             f'{event.metadata.creation_timestamp} '
+                             f'{event.reason} {event.message}')
             except db_utils.UniqueConstraintViolationError:
                 insert_new_pod_event = False
             else:
                 new_event_inserted = True
+                inserted_pod_events += 1
+
+    logger.debug(f'[pod {pod_name}] processed {len(pod_events)} pod events and '
+                 f'inserted {inserted_pod_events} new pod events '
+                 'previously unseen')
 
     if last_scheduled_node is not None:
         node_field_selector = ('involvedObject.kind=Node,'
@@ -1331,6 +1362,7 @@ def _get_pod_missing_reason(context: Optional[str], namespace: str,
             # latest event appears first
             reverse=True)
         insert_new_node_event = True
+        inserted_node_events = 0
         for event in node_events:
             if insert_new_node_event:
                 # Try inserting the latest events first. If the event is a
@@ -1345,10 +1377,23 @@ def _get_pod_missing_reason(context: Optional[str], namespace: str,
                         transitioned_at=int(
                             event.metadata.creation_timestamp.timestamp()),
                         expose_duplicate_error=True)
+                    logger.debug(
+                        f'[pod {pod_name}] encountered new node event: '
+                        f'{event.metadata.creation_timestamp} '
+                        f'{event.reason} {event.message}')
                 except db_utils.UniqueConstraintViolationError:
                     insert_new_node_event = False
                 else:
                     new_event_inserted = True
+                    inserted_node_events += 1
+
+        logger.debug(f'[pod {pod_name}: node {last_scheduled_node}] '
+                     f'processed {len(node_events)} node events and '
+                     f'inserted {inserted_node_events} new node events '
+                     'previously unseen')
+    else:
+        logger.debug(f'[pod {pod_name}] could not determine the node '
+                     'the pod was scheduled to')
 
     if not new_event_inserted:
         # If new event is not inserted, there is no useful information to
@@ -1390,13 +1435,15 @@ def query_instances(
     provider_config: Optional[Dict[str, Any]] = None,
     non_terminated_only: bool = True
 ) -> Dict[str, Tuple[Optional['status_lib.ClusterStatus'], Optional[str]]]:
+    # Mapping from pod phase to skypilot status. These are the only valid pod
+    # phases.
+    # https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase
     status_map = {
         'Pending': status_lib.ClusterStatus.INIT,
         'Running': status_lib.ClusterStatus.UP,
         'Failed': status_lib.ClusterStatus.INIT,
         'Unknown': None,
         'Succeeded': None,
-        'Terminating': None,
     }
 
     assert provider_config is not None
@@ -1440,18 +1487,15 @@ def query_instances(
     for pod in pods:
         phase = pod.status.phase
         pod_status = status_map[phase]
+        reason = None
+        if phase in ('Failed', 'Unknown'):
+            reason = _get_pod_termination_reason(pod, cluster_name)
+            logger.debug(f'Pod Status ({phase}) Reason(s): {reason}')
         if non_terminated_only and pod_status is None:
             logger.debug(f'Pod {pod.metadata.name} is terminated, but '
                          'query_instances is called with '
                          f'non_terminated_only=True. Phase: {phase}')
-            if phase == 'Failed':
-                reason_for_debug = _get_pod_termination_reason(pod)
-                logger.debug(f'Termination reason: {reason_for_debug}')
             continue
-        reason = None
-        if phase == 'Failed':
-            reason = _get_pod_termination_reason(pod)
-            logger.debug(f'Pod Status Reason(s): {reason}')
         pod_name = pod.metadata.name
         reason = f'{pod_name}: {reason}' if reason is not None else None
         cluster_status[pod_name] = (pod_status, reason)

sky/provision/kubernetes/utils.py

@@ -2782,7 +2782,7 @@ def combine_pod_config_fields(
         kubernetes_config)
 
     # Write the updated YAML back to the file
-    common_utils.dump_yaml(cluster_yaml_path, yaml_obj)
+    yaml_utils.dump_yaml(cluster_yaml_path, yaml_obj)
 
 
 def combine_metadata_fields(cluster_yaml_path: str,
@@ -2834,7 +2834,7 @@ def combine_metadata_fields(cluster_yaml_path: str,
         config_utils.merge_k8s_configs(destination, custom_metadata)
 
     # Write the updated YAML back to the file
-    common_utils.dump_yaml(cluster_yaml_path, yaml_obj)
+    yaml_utils.dump_yaml(cluster_yaml_path, yaml_obj)
 
 
 def merge_custom_metadata(

sky/provision/nebius/utils.py

@@ -14,6 +14,8 @@ logger = sky_logging.init_logger(__name__)
 
 POLL_INTERVAL = 5
 
+_MAX_OPERATIONS_TO_FETCH = 1000
+
 
 def retry(func):
     """Decorator to retry a function."""
@@ -321,11 +323,43 @@ def launch(cluster_name_on_cloud: str,
                 parent_id=project_id,
                 name=instance_name,
             )))
+        instance_id = instance.metadata.id
         if instance.status.state.name == 'STARTING':
-            instance_id = instance.metadata.id
             break
+
+        # All Instances initially have state=STOPPED and reconciling=True,
+        # so we need to wait until reconciling is False.
+        if instance.status.state.name == 'STOPPED' and \
+                not instance.status.reconciling:
+            next_token = ''
+            total_operations = 0
+            while True:
+                operations_response = nebius.sync_call(
+                    service.list_operations_by_parent(
+                        nebius.compute().ListOperationsByParentRequest(
+                            parent_id=project_id,
+                            page_size=100,
+                            page_token=next_token,
+                        )))
+                total_operations += len(operations_response.operations)
+                for operation in operations_response.operations:
+                    # Find the most recent operation for the instance.
+                    if operation.resource_id == instance_id:
+                        error_msg = operation.description
+                        if operation.status:
+                            error_msg += f' {operation.status.message}'
+                        raise RuntimeError(error_msg)
+                # If we've fetched too many operations, or there are no more
+                # operations to fetch, just raise a generic error.
+                if total_operations > _MAX_OPERATIONS_TO_FETCH or \
+                        not operations_response.next_page_token:
+                    raise RuntimeError(
+                        f'Instance {instance_name} failed to start.')
+                next_token = operations_response.next_page_token
         time.sleep(POLL_INTERVAL)
-        logger.debug(f'Waiting for instance {instance_name} start running.')
+        logger.debug(f'Waiting for instance {instance_name} to start running. '
+                     f'State: {instance.status.state.name}, '
+                     f'Reconciling: {instance.status.reconciling}')
         retry_count += 1
 
     if retry_count == nebius.MAX_RETRIES_TO_INSTANCE_READY:

sky/serve/client/impl.py

@@ -224,10 +224,11 @@ def tail_logs(service_name: str,
         stream=True)
     request_id: server_common.RequestId[None] = server_common.get_request_id(
         response)
-    return sdk.stream_response(request_id=request_id,
-                               response=response,
-                               output_stream=output_stream,
-                               resumable=True)
+    sdk.stream_response(request_id=request_id,
+                        response=response,
+                        output_stream=output_stream,
+                        resumable=True,
+                        get_result=follow)
 
 
 def sync_down_logs(service_name: str,

sky/serve/replica_managers.py

@@ -37,6 +37,7 @@ from sky.utils import env_options
 from sky.utils import resources_utils
 from sky.utils import status_lib
 from sky.utils import ux_utils
+from sky.utils import yaml_utils
 
 if typing.TYPE_CHECKING:
     from sky.serve import service_spec
@@ -79,7 +80,7 @@ def launch_cluster(replica_id: int,
                     f'{cluster_name} with resources override: '
                     f'{resources_override}')
     try:
-        config = common_utils.read_yaml(
+        config = yaml_utils.read_yaml(
             os.path.expanduser(service_task_yaml_path))
         task = task_lib.Task.from_yaml_config(config)
         if resources_override is not None:
@@ -1397,7 +1398,7 @@ class SkyPilotReplicaManager(ReplicaManager):
         # the latest version. This can significantly improve the speed
         # for updating an existing service with only config changes to the
         # service specs, e.g. scale down the service.
-        new_config = common_utils.read_yaml(
+        new_config = yaml_utils.read_yaml(
             os.path.expanduser(service_task_yaml_path))
         # Always create new replicas and scale down old ones when file_mounts
         # are not empty.
@@ -1414,7 +1415,7 @@ class SkyPilotReplicaManager(ReplicaManager):
                 old_service_task_yaml_path = (
                     serve_utils.generate_task_yaml_file_name(
                         self._service_name, info.version))
-                old_config = common_utils.read_yaml(
+                old_config = yaml_utils.read_yaml(
                     os.path.expanduser(old_service_task_yaml_path))
                 for key in ['service', 'pool', '_user_specified_yaml']:
                     old_config.pop(key, None)

sky/serve/serve_utils.py

@@ -699,7 +699,7 @@ def _get_service_status(
     if record['pool']:
         latest_yaml_path = generate_task_yaml_file_name(service_name,
                                                         record['version'])
-        raw_yaml_config = common_utils.read_yaml(latest_yaml_path)
+        raw_yaml_config = yaml_utils.read_yaml(latest_yaml_path)
         original_config = raw_yaml_config.get('_user_specified_yaml')
         if original_config is None:
             # Fall back to old display format.
@@ -711,7 +711,7 @@ def _get_service_status(
                 original_config['pool'] = svc  # Add pool to root config
         else:
             original_config = yaml_utils.safe_load(original_config)
-        record['pool_yaml'] = common_utils.dump_yaml_str(original_config)
+        record['pool_yaml'] = yaml_utils.dump_yaml_str(original_config)
 
     record['target_num_replicas'] = 0
     try:

sky/serve/server/impl.py

@@ -34,6 +34,7 @@ from sky.utils import dag_utils
 from sky.utils import rich_utils
 from sky.utils import subprocess_utils
 from sky.utils import ux_utils
+from sky.utils import yaml_utils
 
 logger = sky_logging.init_logger(__name__)
 
@@ -179,7 +180,7 @@ def up(
         controller = controller_utils.get_controller_for_pool(pool)
         controller_name = controller.value.cluster_name
         task_config = task.to_yaml_config()
-        common_utils.dump_yaml(service_file.name, task_config)
+        yaml_utils.dump_yaml(service_file.name, task_config)
         remote_tmp_task_yaml_path = (
             serve_utils.generate_remote_tmp_task_yaml_file_name(service_name))
         remote_config_yaml_path = (
@@ -531,7 +532,7 @@ def update(
             prefix=f'{service_name}-v{current_version}',
             mode='w') as service_file:
         task_config = task.to_yaml_config()
-        common_utils.dump_yaml(service_file.name, task_config)
+        yaml_utils.dump_yaml(service_file.name, task_config)
         remote_task_yaml_path = serve_utils.generate_task_yaml_file_name(
             service_name, current_version, expand_user=False)
 

sky/server/auth/oauth2_proxy.py

@@ -4,6 +4,7 @@ import asyncio
 import hashlib
 import http
 import os
+import traceback
 from typing import Optional
 import urllib
 
@@ -109,8 +110,8 @@ class OAuth2ProxyMiddleware(starlette.middleware.base.BaseHTTPMiddleware):
             try:
                 return await self._authenticate(request, call_next, session)
             except (aiohttp.ClientError, asyncio.TimeoutError) as e:
-                logger.error(f'Error communicating with OAuth2 proxy: {e}')
-                # Fail open or closed based on your security requirements
+                logger.error(f'Error communicating with OAuth2 proxy: {e}'
+                             f'{traceback.format_exc()}')
                 return fastapi.responses.JSONResponse(
                     status_code=http.HTTPStatus.BAD_GATEWAY,
                     content={'detail': 'oauth2-proxy service unavailable'})
@@ -120,10 +121,15 @@ class OAuth2ProxyMiddleware(starlette.middleware.base.BaseHTTPMiddleware):
         forwarded_headers = dict(request.headers)
         auth_url = f'{self.proxy_base}/oauth2/auth'
         forwarded_headers['X-Forwarded-Uri'] = str(request.url).rstrip('/')
-        logger.debug(f'authenticate request: {request.url.path}')
+        # Remove content-length and content-type headers and drop request body
+        # to reduce the auth overhead.
+        forwarded_headers.pop('content-length', None)
+        forwarded_headers.pop('content-type', None)
+        logger.debug(f'authenticate request: {auth_url}, '
+                     f'headers: {forwarded_headers}')
 
         async with session.request(
-                method=request.method,
+                method='GET',
                 url=auth_url,
                 headers=forwarded_headers,
                 cookies=request.cookies,

sky/server/common.py

@@ -41,6 +41,7 @@ from sky.utils import annotations
 from sky.utils import common_utils
 from sky.utils import rich_utils
 from sky.utils import ux_utils
+from sky.utils import yaml_utils
 
 if typing.TYPE_CHECKING:
     import aiohttp
@@ -816,7 +817,7 @@ def process_mounts_in_task_on_api_server(task: str, env_vars: Dict[str, str],
         return str(client_file_mounts_dir /
                    file_mounts_mapping[original_path].lstrip('/'))
 
-    task_configs = common_utils.read_yaml_all(str(client_task_path))
+    task_configs = yaml_utils.read_yaml_all(str(client_task_path))
     for task_config in task_configs:
         if task_config is None:
             continue
@@ -869,7 +870,7 @@ def process_mounts_in_task_on_api_server(task: str, env_vars: Dict[str, str],
     # We can switch to using string, but this is to make it easier to debug, by
     # persisting the translated task yaml file.
     translated_client_task_path = client_dir / f'{task_id}_translated.yaml'
-    common_utils.dump_yaml(str(translated_client_task_path), task_configs)
+    yaml_utils.dump_yaml(str(translated_client_task_path), task_configs)
 
     dag = dag_utils.load_chain_dag_from_yaml(str(translated_client_task_path))
     return dag

sky/server/daemons.py

@@ -191,23 +191,28 @@ INTERNAL_REQUEST_DAEMONS = [
     # set to updated status automatically, without showing users the hint of
     # cluster being stopped or down when `sky status -r` is called.
     InternalRequestDaemon(id='skypilot-status-refresh-daemon',
-                          name='status',
+                          name='status-refresh',
                           event_fn=refresh_cluster_status_event,
                           default_log_level='DEBUG'),
     # Volume status refresh daemon to update the volume status periodically.
     InternalRequestDaemon(id='skypilot-volume-status-refresh-daemon',
-                          name='volume',
+                          name='volume-refresh',
                           event_fn=refresh_volume_status_event),
     InternalRequestDaemon(id='managed-job-status-refresh-daemon',
-                          name='managed-job-status',
+                          name='managed-job-status-refresh',
                           event_fn=managed_job_status_refresh_event,
                           should_skip=should_skip_managed_job_status_refresh),
     InternalRequestDaemon(id='sky-serve-status-refresh-daemon',
-                          name='sky-serve-status',
+                          name='sky-serve-status-refresh',
                           event_fn=sky_serve_status_refresh_event,
                           should_skip=should_skip_sky_serve_status_refresh),
     InternalRequestDaemon(id='pool-status-refresh-daemon',
-                          name='pool-status',
+                          name='pool-status-refresh',
                           event_fn=pool_status_refresh_event,
                           should_skip=should_skip_pool_status_refresh),
 ]
+
+
+def is_daemon_request_id(request_id: str) -> bool:
+    """Returns whether a specific request_id is an internal daemon."""
+    return any([d.id == request_id for d in INTERNAL_REQUEST_DAEMONS])

sky/server/requests/executor.py

@@ -55,6 +55,7 @@ from sky.utils import context_utils
 from sky.utils import subprocess_utils
 from sky.utils import tempstore
 from sky.utils import timeline
+from sky.utils import yaml_utils
 from sky.workspaces import core as workspaces_core
 
 if typing.TYPE_CHECKING:
@@ -387,7 +388,7 @@ def _request_execution_wrapper(request_id: str,
                 if sky_logging.logging_enabled(logger, sky_logging.DEBUG):
                     config = skypilot_config.to_dict()
                     logger.debug(f'request config: \n'
-                                 f'{common_utils.dump_yaml_str(dict(config))}')
+                                 f'{yaml_utils.dump_yaml_str(dict(config))}')
                 return_value = func(**request_body.to_kwargs())
                 f.flush()
         except KeyboardInterrupt:

sky/server/requests/requests.py

@@ -565,6 +565,27 @@ def get_request_tasks(
     return requests
 
 
+@init_db
+def get_api_request_ids_start_with(incomplete: str) -> List[str]:
+    """Get a list of API request ids for shell completion."""
+    assert _DB is not None
+    with _DB.conn:
+        cursor = _DB.conn.cursor()
+        # Prioritize alive requests (PENDING, RUNNING) over finished ones,
+        # then order by creation time (newest first) within each category.
+        cursor.execute(
+            f"""SELECT request_id FROM {REQUEST_TABLE}
+                WHERE request_id LIKE ?
+                ORDER BY
+                    CASE
+                        WHEN status IN ('PENDING', 'RUNNING') THEN 0
+                        ELSE 1
+                    END,
+                    created_at DESC
+                LIMIT 1000""", (f'{incomplete}%',))
+        return [row[0] for row in cursor.fetchall()]
+
+
 def _add_or_update_request_no_lock(request: Request):
     """Add or update a REST request into the database."""
     row = request.to_row()

sky/server/server.py

@@ -1403,6 +1403,9 @@ async def api_get(request_id: str) -> payloads.RequestPayload:
                 raise fastapi.HTTPException(
                     status_code=500, detail=request_task.encode().model_dump())
             return request_task.encode()
+        elif (request_task.status == requests_lib.RequestStatus.RUNNING and
+              daemons.is_daemon_request_id(request_id)):
+            return request_task.encode()
         # yield control to allow other coroutines to run, sleep shortly
         # to avoid storming the DB and CPU in the meantime
         await asyncio.sleep(0.1)
@@ -1491,6 +1494,14 @@ async def stream(
         if log_path == constants.API_SERVER_LOGS:
             resolved_log_path = pathlib.Path(
                 constants.API_SERVER_LOGS).expanduser()
+            if not resolved_log_path.exists():
+                raise fastapi.HTTPException(
+                    status_code=404,
+                    detail='Server log file does not exist. The API server may '
+                    'have been started with `--foreground` - check the '
+                    'stdout of API server process, such as: '
+                    '`kubectl logs -n api-server-namespace '
+                    'api-server-pod-name`')
         else:
             # This should be a log path under ~/sky_logs.
             resolved_logs_directory = pathlib.Path(
@@ -1769,6 +1780,11 @@ async def complete_volume_name(incomplete: str,) -> List[str]:
     return global_user_state.get_volume_names_start_with(incomplete)
 
 
+@app.get('/api/completion/api_request')
+async def complete_api_request(incomplete: str,) -> List[str]:
+    return requests_lib.get_api_request_ids_start_with(incomplete)
+
+
 @app.get('/dashboard/{full_path:path}')
 async def serve_dashboard(full_path: str):
     """Serves the Next.js dashboard application.

sky/skylet/events.py

@@ -20,7 +20,6 @@ from sky.skylet import constants
 from sky.skylet import job_lib
 from sky.usage import usage_lib
 from sky.utils import cluster_utils
-from sky.utils import common_utils
 from sky.utils import registry
 from sky.utils import ux_utils
 from sky.utils import yaml_utils
@@ -181,7 +180,7 @@ class AutostopEvent(SkyletEvent):
 
             config_path = os.path.abspath(
                 os.path.expanduser(cluster_utils.SKY_CLUSTER_YAML_REMOTE_PATH))
-            config = common_utils.read_yaml(config_path)
+            config = yaml_utils.read_yaml(config_path)
             provider_name = cluster_utils.get_provider_name(config)
             cloud = registry.CLOUD_REGISTRY.from_str(provider_name)
             assert cloud is not None, f'Unknown cloud: {provider_name}'
@@ -326,5 +325,5 @@ class AutostopEvent(SkyletEvent):
         config['auth'].pop('ssh_proxy_command', None)
         # Empty the file_mounts.
         config['file_mounts'] = {}
-        common_utils.dump_yaml(yaml_path, config)
+        yaml_utils.dump_yaml(yaml_path, config)
         logger.debug('Replaced upscaling speed to 0.')

sky/skypilot_config.py

@@ -494,7 +494,7 @@ def reload_config() -> None:
 def parse_and_validate_config_file(config_path: str) -> config_utils.Config:
     config = config_utils.Config()
     try:
-        config_dict = common_utils.read_yaml(config_path)
+        config_dict = yaml_utils.read_yaml(config_path)
         config = config_utils.Config.from_dict(config_dict)
         # pop the db url from the config, and set it to the env var.
         # this is to avoid db url (considered a sensitive value)
@@ -504,7 +504,7 @@ def parse_and_validate_config_file(config_path: str) -> config_utils.Config:
             os.environ[constants.ENV_VAR_DB_CONNECTION_URI] = db_url
         if sky_logging.logging_enabled(logger, sky_logging.DEBUG):
             logger.debug(f'Config loaded from {config_path}:\n'
-                         f'{common_utils.dump_yaml_str(dict(config))}')
+                         f'{yaml_utils.dump_yaml_str(dict(config))}')
     except yaml.YAMLError as e:
         logger.error(f'Error in loading config file ({config_path}):', e)
     if config:
@@ -600,7 +600,7 @@ def _reload_config_as_server() -> None:
             sqlalchemy_engine.dispose()
     if sky_logging.logging_enabled(logger, sky_logging.DEBUG):
         logger.debug(f'server config: \n'
-                     f'{common_utils.dump_yaml_str(dict(server_config))}')
+                     f'{yaml_utils.dump_yaml_str(dict(server_config))}')
     _set_loaded_config(server_config)
     _set_loaded_config_path(server_config_path)
 
@@ -628,7 +628,7 @@ def _reload_config_as_client() -> None:
     if sky_logging.logging_enabled(logger, sky_logging.DEBUG):
         logger.debug(
             f'client config (before task and CLI overrides): \n'
-            f'{common_utils.dump_yaml_str(dict(overlaid_client_config))}')
+            f'{yaml_utils.dump_yaml_str(dict(overlaid_client_config))}')
     _set_loaded_config(overlaid_client_config)
     _set_loaded_config_path([user_config_path, project_config_path])
 
@@ -738,9 +738,9 @@ def override_skypilot_config(
                 'Failed to override the SkyPilot config on API '
                 'server with your local SkyPilot config:\n'
                 '=== SkyPilot config on API server ===\n'
-                f'{common_utils.dump_yaml_str(dict(original_config))}\n'
+                f'{yaml_utils.dump_yaml_str(dict(original_config))}\n'
                 '=== Your local SkyPilot config ===\n'
-                f'{common_utils.dump_yaml_str(dict(override_configs))}\n'
+                f'{yaml_utils.dump_yaml_str(dict(override_configs))}\n'
                 f'Details: {e}') from e
     finally:
         _set_loaded_config(original_config)
@@ -767,7 +767,7 @@ def replace_skypilot_config(new_configs: config_utils.Config) -> Iterator[None]:
                                          mode='w',
                                          prefix='mutated-skypilot-config-',
                                          suffix='.yaml') as temp_file:
-            common_utils.dump_yaml(temp_file.name, dict(**new_configs))
+            yaml_utils.dump_yaml(temp_file.name, dict(**new_configs))
         # Modify the env var of current process or context so that the
         # new config will be used by spawned sub-processes.
         # Note that this code modifies os.environ directly because it
@@ -831,7 +831,7 @@ def apply_cli_config(cli_config: Optional[List[str]]) -> Dict[str, Any]:
     parsed_config = _compose_cli_config(cli_config)
     if sky_logging.logging_enabled(logger, sky_logging.DEBUG):
         logger.debug(f'applying following CLI overrides: \n'
-                     f'{common_utils.dump_yaml_str(dict(parsed_config))}')
+                     f'{yaml_utils.dump_yaml_str(dict(parsed_config))}')
     _set_loaded_config(
         overlay_skypilot_config(original_config=_get_loaded_config(),
                                 override_configs=parsed_config))
@@ -875,7 +875,7 @@ def update_api_server_config_no_lock(config: config_utils.Config) -> None:
                 def _set_config_yaml_to_db(key: str,
                                            config: config_utils.Config):
                     assert sqlalchemy_engine is not None
-                    config_str = common_utils.dump_yaml_str(dict(config))
+                    config_str = yaml_utils.dump_yaml_str(dict(config))
                     with orm.Session(sqlalchemy_engine) as session:
                         if (sqlalchemy_engine.dialect.name ==
                                 db_utils.SQLAlchemyDialect.SQLITE.value):
@@ -901,7 +901,7 @@ def update_api_server_config_no_lock(config: config_utils.Config) -> None:
 
     if not db_updated:
         # save to the local file (PVC in Kubernetes, local file otherwise)
-        common_utils.dump_yaml(global_config_path, dict(config))
+        yaml_utils.dump_yaml(global_config_path, dict(config))
 
         if config_map_utils.is_running_in_kubernetes():
             # In Kubernetes, sync the PVC config to ConfigMap for user

sky/task.py

@@ -564,7 +564,7 @@ class Task:
         secrets_overrides: Optional[List[Tuple[str, str]]] = None,
     ) -> 'Task':
         user_specified_yaml = config.pop('_user_specified_yaml',
-                                         common_utils.dump_yaml_str(config))
+                                         yaml_utils.dump_yaml_str(config))
         # More robust handling for 'envs': explicitly convert keys and values to
         # str, since users may pass '123' as keys/values which will get parsed
         # as int causing validate_schema() to fail.

sky/templates/nebius-ray.yml.j2

@@ -56,15 +56,11 @@ available_node_types:
           filesystem_mount_path: {{ fs.filesystem_mount_path }}
         {%- endfor %}
       UserData: |
-        runcmd:
-          - sudo sed -i 's/^#\?AllowTcpForwarding.*/AllowTcpForwarding yes/' /etc/ssh/sshd_config
-          - systemctl restart sshd
-
         {# Two available OS images:
-               1. ubuntu22.04-driverless - requires Docker installation
-               2. ubuntu22.04-cuda12 - comes with Docker pre-installed
-            To optimize deployment speed, Docker is only installed when using ubuntu22.04-driverless #}
-        {%- if docker_image is not none and image_id == 'ubuntu22.04-driverless' %}
+               1. ubuntu24.04-driverless - requires Docker installation
+               2. ubuntu24.04-cuda12 - comes with Docker pre-installed
+            To optimize deployment speed, Docker is only installed when using ubuntu24.04-driverless #}
+        {%- if docker_image is not none and image_id.endswith('-driverless') %}
         apt:
           sources:
             docker.list: