skypilot-nightly 1.0.0.dev20250826__py3-none-any.whl → 1.0.0.dev20250828__py3-none-any.whl

sky/provision/kubernetes/utils.py

@@ -38,6 +38,7 @@ from sky.utils import schemas
 from sky.utils import status_lib
 from sky.utils import timeline
 from sky.utils import ux_utils
+from sky.utils import yaml_utils
 
 if typing.TYPE_CHECKING:
     import jinja2
@@ -1898,7 +1899,7 @@ def is_kubeconfig_exec_auth(
 
     # Load the kubeconfig for the context
     kubeconfig_text = _get_kubeconfig_text_for_context(context)
-    kubeconfig = yaml.safe_load(kubeconfig_text)
+    kubeconfig = yaml_utils.safe_load(kubeconfig_text)
 
     # Get the user details
     user_details = kubeconfig['users']
@@ -2601,7 +2602,7 @@ def fill_ssh_jump_template(ssh_key_secret: str, ssh_jump_image: str,
                               image=ssh_jump_image,
                               secret=ssh_key_secret,
                               service_type=service_type)
-    content = yaml.safe_load(cont)
+    content = yaml_utils.safe_load(cont)
     return content
 
 
@@ -2750,7 +2751,7 @@ def combine_pod_config_fields(
     """
     with open(cluster_yaml_path, 'r', encoding='utf-8') as f:
         yaml_content = f.read()
-    yaml_obj = yaml.safe_load(yaml_content)
+    yaml_obj = yaml_utils.safe_load(yaml_content)
     # We don't use override_configs in `get_effective_region_config`, as merging
     # the pod config requires special handling.
     if isinstance(cloud, clouds.SSH):
@@ -2781,7 +2782,7 @@ def combine_pod_config_fields(
         kubernetes_config)
 
     # Write the updated YAML back to the file
-    common_utils.dump_yaml(cluster_yaml_path, yaml_obj)
+    yaml_utils.dump_yaml(cluster_yaml_path, yaml_obj)
 
 
 def combine_metadata_fields(cluster_yaml_path: str,
@@ -2795,7 +2796,7 @@ def combine_metadata_fields(cluster_yaml_path: str,
 
     with open(cluster_yaml_path, 'r', encoding='utf-8') as f:
         yaml_content = f.read()
-    yaml_obj = yaml.safe_load(yaml_content)
+    yaml_obj = yaml_utils.safe_load(yaml_content)
 
     # Get custom_metadata from global config
     custom_metadata = skypilot_config.get_effective_region_config(
@@ -2833,7 +2834,7 @@ def combine_metadata_fields(cluster_yaml_path: str,
         config_utils.merge_k8s_configs(destination, custom_metadata)
 
     # Write the updated YAML back to the file
-    common_utils.dump_yaml(cluster_yaml_path, yaml_obj)
+    yaml_utils.dump_yaml(cluster_yaml_path, yaml_obj)
 
 
 def merge_custom_metadata(
@@ -3689,7 +3690,7 @@ def format_kubeconfig_exec_auth_with_cache(kubeconfig_path: str) -> str:
     """
     # TODO(kyuds): GC cache files
     with open(kubeconfig_path, 'r', encoding='utf-8') as file:
-        config = yaml.safe_load(file)
+        config = yaml_utils.safe_load(file)
     normalized = yaml.dump(config, sort_keys=True)
     hashed = hashlib.sha1(normalized.encode('utf-8')).hexdigest()
     path = os.path.expanduser(

sky/provision/nebius/utils.py

@@ -14,6 +14,8 @@ logger = sky_logging.init_logger(__name__)
 
 POLL_INTERVAL = 5
 
+_MAX_OPERATIONS_TO_FETCH = 1000
+
 
 def retry(func):
     """Decorator to retry a function."""
@@ -100,15 +102,23 @@ def delete_cluster(name: str, region: str) -> None:
 def list_instances(project_id: str) -> Dict[str, Dict[str, Any]]:
     """Lists instances associated with API key."""
     service = nebius.compute().InstanceServiceClient(nebius.sdk())
-    result = nebius.sync_call(
-        service.list(
-            nebius.compute().ListInstancesRequest(parent_id=project_id),
-            timeout=nebius.READ_TIMEOUT))
-
-    instances = result
+    page_token = ''
+    instances = []
+    while True:
+        result = nebius.sync_call(
+            service.list(nebius.compute().ListInstancesRequest(
+                parent_id=project_id,
+                page_size=100,
+                page_token=page_token,
+            ),
+                         timeout=nebius.READ_TIMEOUT))
+        instances.extend(result.items)
+        if not result.next_page_token:  # "" means no more pages
+            break
+        page_token = result.next_page_token
 
     instance_dict: Dict[str, Dict[str, Any]] = {}
-    for instance in instances.items:
+    for instance in instances:
         info = {}
         info['status'] = instance.status.state.name
         info['name'] = instance.metadata.name
@@ -313,11 +323,43 @@ def launch(cluster_name_on_cloud: str,
                 parent_id=project_id,
                 name=instance_name,
             )))
+        instance_id = instance.metadata.id
         if instance.status.state.name == 'STARTING':
-            instance_id = instance.metadata.id
             break
+
+        # All Instances initially have state=STOPPED and reconciling=True,
+        # so we need to wait until reconciling is False.
+        if instance.status.state.name == 'STOPPED' and \
+                not instance.status.reconciling:
+            next_token = ''
+            total_operations = 0
+            while True:
+                operations_response = nebius.sync_call(
+                    service.list_operations_by_parent(
+                        nebius.compute().ListOperationsByParentRequest(
+                            parent_id=project_id,
+                            page_size=100,
+                            page_token=next_token,
+                        )))
+                total_operations += len(operations_response.operations)
+                for operation in operations_response.operations:
+                    # Find the most recent operation for the instance.
+                    if operation.resource_id == instance_id:
+                        error_msg = operation.description
+                        if operation.status:
+                            error_msg += f' {operation.status.message}'
+                        raise RuntimeError(error_msg)
+                # If we've fetched too many operations, or there are no more
+                # operations to fetch, just raise a generic error.
+                if total_operations > _MAX_OPERATIONS_TO_FETCH or \
+                        not operations_response.next_page_token:
+                    raise RuntimeError(
+                        f'Instance {instance_name} failed to start.')
+                next_token = operations_response.next_page_token
         time.sleep(POLL_INTERVAL)
-        logger.debug(f'Waiting for instance {instance_name} start running.')
+        logger.debug(f'Waiting for instance {instance_name} to start running. '
+                     f'State: {instance.status.state.name}, '
+                     f'Reconciling: {instance.status.reconciling}')
         retry_count += 1
 
     if retry_count == nebius.MAX_RETRIES_TO_INSTANCE_READY:

sky/provision/vsphere/vsphere_utils.py

@@ -3,12 +3,10 @@
 import http.cookies as http_cookies
 import os
 import ssl
-import typing
 from typing import Any, Dict, List, Optional
 
 from sky import exceptions
 from sky import sky_logging
-from sky.adaptors import common as adaptors_common
 from sky.adaptors import vsphere as vsphere_adaptor
 from sky.catalog import vsphere_catalog
 from sky.catalog.common import get_catalog_path
@@ -28,11 +26,7 @@ from sky.provision.vsphere.common.vim_utils import create_spec_with_script
 from sky.provision.vsphere.common.vim_utils import poweron_vm
 from sky.provision.vsphere.common.vim_utils import wait_for_tasks
 from sky.provision.vsphere.common.vim_utils import wait_internal_ip_ready
-
-if typing.TYPE_CHECKING:
-    import yaml
-else:
-    yaml = adaptors_common.LazyImport('yaml')
+from sky.utils import yaml_utils
 
 logger = sky_logging.init_logger(__name__)
 
@@ -323,7 +317,7 @@ def get_vsphere_credentials(name=None):
     assert os.path.exists(
         credential_path), f'Missing credential file at {credential_path}.'
     with open(credential_path, 'r', encoding='utf-8') as file:
-        credential = yaml.safe_load(file)
+        credential = yaml_utils.safe_load(file)
         vcenters = credential['vcenters']
         if name is None:
             return vcenters

sky/schemas/api/responses.py

@@ -1,5 +1,6 @@
 """Responses for the API server."""
 
+import enum
 from typing import Any, Dict, List, Optional
 
 import pydantic
@@ -117,3 +118,9 @@ class StatusResponse(ResponseBaseModel):
     cpus: Optional[str] = None
     memory: Optional[str] = None
     accelerators: Optional[str] = None
+
+
+class UploadStatus(enum.Enum):
+    """Status of the upload."""
+    UPLOADING = 'uploading'
+    COMPLETED = 'completed'

sky/serve/client/impl.py

@@ -224,10 +224,11 @@ def tail_logs(service_name: str,
         stream=True)
     request_id: server_common.RequestId[None] = server_common.get_request_id(
         response)
-    return sdk.stream_response(request_id=request_id,
-                               response=response,
-                               output_stream=output_stream,
-                               resumable=True)
+    sdk.stream_response(request_id=request_id,
+                        response=response,
+                        output_stream=output_stream,
+                        resumable=True,
+                        get_result=follow)
 
 
 def sync_down_logs(service_name: str,

sky/serve/replica_managers.py

@@ -37,6 +37,7 @@ from sky.utils import env_options
 from sky.utils import resources_utils
 from sky.utils import status_lib
 from sky.utils import ux_utils
+from sky.utils import yaml_utils
 
 if typing.TYPE_CHECKING:
     from sky.serve import service_spec
@@ -79,7 +80,7 @@ def launch_cluster(replica_id: int,
                     f'{cluster_name} with resources override: '
                     f'{resources_override}')
     try:
-        config = common_utils.read_yaml(
+        config = yaml_utils.read_yaml(
             os.path.expanduser(service_task_yaml_path))
         task = task_lib.Task.from_yaml_config(config)
         if resources_override is not None:
@@ -1397,7 +1398,7 @@ class SkyPilotReplicaManager(ReplicaManager):
         # the latest version. This can significantly improve the speed
         # for updating an existing service with only config changes to the
         # service specs, e.g. scale down the service.
-        new_config = common_utils.read_yaml(
+        new_config = yaml_utils.read_yaml(
             os.path.expanduser(service_task_yaml_path))
         # Always create new replicas and scale down old ones when file_mounts
         # are not empty.
@@ -1414,7 +1415,7 @@ class SkyPilotReplicaManager(ReplicaManager):
                 old_service_task_yaml_path = (
                     serve_utils.generate_task_yaml_file_name(
                         self._service_name, info.version))
-                old_config = common_utils.read_yaml(
+                old_config = yaml_utils.read_yaml(
                     os.path.expanduser(old_service_task_yaml_path))
                 for key in ['service', 'pool', '_user_specified_yaml']:
                     old_config.pop(key, None)

sky/serve/serve_utils.py

@@ -20,7 +20,6 @@ import uuid
 
 import colorama
 import filelock
-import yaml
 
 from sky import backends
 from sky import exceptions
@@ -43,6 +42,7 @@ from sky.utils import message_utils
 from sky.utils import resources_utils
 from sky.utils import status_lib
 from sky.utils import ux_utils
+from sky.utils import yaml_utils
 
 if typing.TYPE_CHECKING:
     import fastapi
@@ -699,7 +699,7 @@ def _get_service_status(
     if record['pool']:
         latest_yaml_path = generate_task_yaml_file_name(service_name,
                                                         record['version'])
-        raw_yaml_config = common_utils.read_yaml(latest_yaml_path)
+        raw_yaml_config = yaml_utils.read_yaml(latest_yaml_path)
         original_config = raw_yaml_config.get('_user_specified_yaml')
         if original_config is None:
             # Fall back to old display format.
@@ -710,8 +710,8 @@ def _get_service_status(
                 svc.pop('pool', None)  # Remove pool from service config
                 original_config['pool'] = svc  # Add pool to root config
         else:
-            original_config = yaml.safe_load(original_config)
-        record['pool_yaml'] = common_utils.dump_yaml_str(original_config)
+            original_config = yaml_utils.safe_load(original_config)
+        record['pool_yaml'] = yaml_utils.dump_yaml_str(original_config)
 
     record['target_num_replicas'] = 0
     try:

sky/serve/server/impl.py

@@ -34,6 +34,7 @@ from sky.utils import dag_utils
 from sky.utils import rich_utils
 from sky.utils import subprocess_utils
 from sky.utils import ux_utils
+from sky.utils import yaml_utils
 
 logger = sky_logging.init_logger(__name__)
 
@@ -179,7 +180,7 @@ def up(
         controller = controller_utils.get_controller_for_pool(pool)
         controller_name = controller.value.cluster_name
         task_config = task.to_yaml_config()
-        common_utils.dump_yaml(service_file.name, task_config)
+        yaml_utils.dump_yaml(service_file.name, task_config)
         remote_tmp_task_yaml_path = (
             serve_utils.generate_remote_tmp_task_yaml_file_name(service_name))
         remote_config_yaml_path = (
@@ -531,7 +532,7 @@ def update(
             prefix=f'{service_name}-v{current_version}',
             mode='w') as service_file:
         task_config = task.to_yaml_config()
-        common_utils.dump_yaml(service_file.name, task_config)
+        yaml_utils.dump_yaml(service_file.name, task_config)
         remote_task_yaml_path = serve_utils.generate_task_yaml_file_name(
             service_name, current_version, expand_user=False)
 

sky/serve/service_spec.py

@@ -2,11 +2,9 @@
 import json
 import os
 import textwrap
-import typing
 from typing import Any, Dict, List, Optional, Union
 
 from sky import serve
-from sky.adaptors import common as adaptors_common
 from sky.serve import constants
 from sky.serve import load_balancing_policies as lb_policies
 from sky.serve import serve_utils
@@ -14,11 +12,7 @@ from sky.serve import spot_placer as spot_placer_lib
 from sky.utils import common_utils
 from sky.utils import schemas
 from sky.utils import ux_utils
-
-if typing.TYPE_CHECKING:
-    import yaml
-else:
-    yaml = adaptors_common.LazyImport('yaml')
+from sky.utils import yaml_utils
 
 
 class SkyServiceSpec:
@@ -274,7 +268,7 @@ class SkyServiceSpec:
     @staticmethod
     def from_yaml(yaml_path: str) -> 'SkyServiceSpec':
         with open(os.path.expanduser(yaml_path), 'r', encoding='utf-8') as f:
-            config = yaml.safe_load(f)
+            config = yaml_utils.safe_load(f)
 
         if isinstance(config, str):
             with ux_utils.print_exception_no_traceback():

sky/server/auth/authn.py

@@ -14,6 +14,10 @@ logger = sky_logging.init_logger(__name__)
 # TODO(hailong): Remove this function and use request.state.auth_user instead.
 async def override_user_info_in_request_body(request: fastapi.Request,
                                              auth_user: Optional[models.User]):
+    # Skip for upload requests to avoid consuming the body prematurely, which
+    # will break the streaming upload.
+    if request.url.path.startswith('/upload'):
+        return
     if auth_user is None:
         return
 

sky/server/auth/oauth2_proxy.py

@@ -4,6 +4,7 @@ import asyncio
 import hashlib
 import http
 import os
+import traceback
 from typing import Optional
 import urllib
 
@@ -109,8 +110,8 @@ class OAuth2ProxyMiddleware(starlette.middleware.base.BaseHTTPMiddleware):
             try:
                 return await self._authenticate(request, call_next, session)
             except (aiohttp.ClientError, asyncio.TimeoutError) as e:
-                logger.error(f'Error communicating with OAuth2 proxy: {e}')
-                # Fail open or closed based on your security requirements
+                logger.error(f'Error communicating with OAuth2 proxy: {e}'
+                             f'{traceback.format_exc()}')
                 return fastapi.responses.JSONResponse(
                     status_code=http.HTTPStatus.BAD_GATEWAY,
                     content={'detail': 'oauth2-proxy service unavailable'})
@@ -120,10 +121,15 @@ class OAuth2ProxyMiddleware(starlette.middleware.base.BaseHTTPMiddleware):
         forwarded_headers = dict(request.headers)
         auth_url = f'{self.proxy_base}/oauth2/auth'
         forwarded_headers['X-Forwarded-Uri'] = str(request.url).rstrip('/')
-        logger.debug(f'authenticate request: {request.url.path}')
+        # Remove content-length and content-type headers and drop request body
+        # to reduce the auth overhead.
+        forwarded_headers.pop('content-length', None)
+        forwarded_headers.pop('content-type', None)
+        logger.debug(f'authenticate request: {auth_url}, '
+                     f'headers: {forwarded_headers}')
 
         async with session.request(
-                method=request.method,
+                method='GET',
                 url=auth_url,
                 headers=forwarded_headers,
                 cookies=request.cookies,

sky/server/common.py

@@ -23,6 +23,7 @@ import uuid
 import cachetools
 import colorama
 import filelock
+from passlib import context as passlib_context
 from typing_extensions import ParamSpec
 
 from sky import exceptions
@@ -40,6 +41,7 @@ from sky.utils import annotations
 from sky.utils import common_utils
 from sky.utils import rich_utils
 from sky.utils import ux_utils
+from sky.utils import yaml_utils
 
 if typing.TYPE_CHECKING:
     import aiohttp
@@ -61,7 +63,7 @@ AVAILABLE_LOCAL_API_SERVER_URLS = [
 
 API_SERVER_CMD = '-m sky.server.server'
 # The client dir on the API server for storing user-specific data, such as file
-# mounts, logs, etc. This dir is empheral and will be cleaned up when the API
+# mounts, logs, etc. This dir is ephemeral and will be cleaned up when the API
 # server is restarted.
 API_SERVER_CLIENT_DIR = pathlib.Path('~/.sky/api_server/clients')
 RETRY_COUNT_ON_TIMEOUT = 3
@@ -102,6 +104,11 @@ logger = sky_logging.init_logger(__name__)
 
 hinted_for_server_install_version_mismatch = False
 
+crypt_ctx = passlib_context.CryptContext([
+    'bcrypt', 'sha256_crypt', 'sha512_crypt', 'des_crypt', 'apr_md5_crypt',
+    'ldap_sha1'
+])
+
 
 class ApiServerStatus(enum.Enum):
     HEALTHY = 'healthy'
@@ -810,7 +817,7 @@ def process_mounts_in_task_on_api_server(task: str, env_vars: Dict[str, str],
         return str(client_file_mounts_dir /
                    file_mounts_mapping[original_path].lstrip('/'))
 
-    task_configs = common_utils.read_yaml_all(str(client_task_path))
+    task_configs = yaml_utils.read_yaml_all(str(client_task_path))
     for task_config in task_configs:
         if task_config is None:
             continue
@@ -863,7 +870,7 @@ def process_mounts_in_task_on_api_server(task: str, env_vars: Dict[str, str],
     # We can switch to using string, but this is to make it easier to debug, by
     # persisting the translated task yaml file.
     translated_client_task_path = client_dir / f'{task_id}_translated.yaml'
-    common_utils.dump_yaml(str(translated_client_task_path), task_configs)
+    yaml_utils.dump_yaml(str(translated_client_task_path), task_configs)
 
     dag = dag_utils.load_chain_dag_from_yaml(str(translated_client_task_path))
     return dag

sky/server/daemons.py

@@ -191,23 +191,28 @@ INTERNAL_REQUEST_DAEMONS = [
     # set to updated status automatically, without showing users the hint of
     # cluster being stopped or down when `sky status -r` is called.
     InternalRequestDaemon(id='skypilot-status-refresh-daemon',
-                          name='status',
+                          name='status-refresh',
                           event_fn=refresh_cluster_status_event,
                           default_log_level='DEBUG'),
     # Volume status refresh daemon to update the volume status periodically.
     InternalRequestDaemon(id='skypilot-volume-status-refresh-daemon',
-                          name='volume',
+                          name='volume-refresh',
                           event_fn=refresh_volume_status_event),
     InternalRequestDaemon(id='managed-job-status-refresh-daemon',
-                          name='managed-job-status',
+                          name='managed-job-status-refresh',
                           event_fn=managed_job_status_refresh_event,
                           should_skip=should_skip_managed_job_status_refresh),
     InternalRequestDaemon(id='sky-serve-status-refresh-daemon',
-                          name='sky-serve-status',
+                          name='sky-serve-status-refresh',
                           event_fn=sky_serve_status_refresh_event,
                           should_skip=should_skip_sky_serve_status_refresh),
     InternalRequestDaemon(id='pool-status-refresh-daemon',
-                          name='pool-status',
+                          name='pool-status-refresh',
                           event_fn=pool_status_refresh_event,
                           should_skip=should_skip_pool_status_refresh),
 ]
+
+
+def is_daemon_request_id(request_id: str) -> bool:
+    """Returns whether a specific request_id is an internal daemon."""
+    return any([d.id == request_id for d in INTERNAL_REQUEST_DAEMONS])

sky/server/requests/executor.py

@@ -55,6 +55,7 @@ from sky.utils import context_utils
 from sky.utils import subprocess_utils
 from sky.utils import tempstore
 from sky.utils import timeline
+from sky.utils import yaml_utils
 from sky.workspaces import core as workspaces_core
 
 if typing.TYPE_CHECKING:
@@ -275,6 +276,10 @@ def override_request_env_and_config(
         request_id: str) -> Generator[None, None, None]:
     """Override the environment and SkyPilot config for a request."""
     original_env = os.environ.copy()
+    # Unset SKYPILOT_DEBUG by default, to avoid the value set on the API server
+    # affecting client requests. If set on the client side, it will be
+    # overridden by the request body.
+    os.environ.pop('SKYPILOT_DEBUG', None)
     os.environ.update(request_body.env_vars)
     # Note: may be overridden by AuthProxyMiddleware.
     # TODO(zhwu): we need to make the entire request a context available to the
@@ -383,7 +388,7 @@ def _request_execution_wrapper(request_id: str,
                 if sky_logging.logging_enabled(logger, sky_logging.DEBUG):
                     config = skypilot_config.to_dict()
                     logger.debug(f'request config: \n'
-                                 f'{common_utils.dump_yaml_str(dict(config))}')
+                                 f'{yaml_utils.dump_yaml_str(dict(config))}')
                 return_value = func(**request_body.to_kwargs())
                 f.flush()
         except KeyboardInterrupt:

sky/server/requests/requests.py

@@ -565,6 +565,27 @@ def get_request_tasks(
     return requests
 
 
+@init_db
+def get_api_request_ids_start_with(incomplete: str) -> List[str]:
+    """Get a list of API request ids for shell completion."""
+    assert _DB is not None
+    with _DB.conn:
+        cursor = _DB.conn.cursor()
+        # Prioritize alive requests (PENDING, RUNNING) over finished ones,
+        # then order by creation time (newest first) within each category.
+        cursor.execute(
+            f"""SELECT request_id FROM {REQUEST_TABLE}
+                WHERE request_id LIKE ?
+                ORDER BY
+                    CASE
+                        WHEN status IN ('PENDING', 'RUNNING') THEN 0
+                        ELSE 1
+                    END,
+                    created_at DESC
+                LIMIT 1000""", (f'{incomplete}%',))
+        return [row[0] for row in cursor.fetchall()]
+
+
 def _add_or_update_request_no_lock(request: Request):
     """Add or update a REST request into the database."""
     row = request.to_row()