skypilot-nightly 1.0.0.dev20250814__py3-none-any.whl → 1.0.0.dev20250815__py3-none-any.whl

sky/server/requests/serializers/decoders.py

@@ -102,8 +102,18 @@ def decode_queue(return_value: List[dict],) -> List[Dict[str, Any]]:
 
 
 @register_decoders('jobs.queue')
-def decode_jobs_queue(return_value: List[dict],) -> List[Dict[str, Any]]:
-    jobs = return_value
+def decode_jobs_queue(return_value):
+    """Decode jobs queue response.
+
+    Supports legacy list, or a dict {jobs, total}.
+    - Returns list[job]
+    """
+    # Case 1: dict shape {jobs, total}
+    if isinstance(return_value, dict) and 'jobs' in return_value:
+        jobs = return_value.get('jobs', [])
+    else:
+        # Case 2: legacy list
+        jobs = return_value
     for job in jobs:
         job['status'] = managed_jobs.ManagedJobStatus(job['status'])
     return jobs

sky/server/requests/serializers/encoders.py

@@ -106,10 +106,18 @@ def encode_status_kubernetes(
 
 
 @register_encoder('jobs.queue')
-def encode_jobs_queue(jobs: List[dict],) -> List[Dict[str, Any]]:
+def encode_jobs_queue(jobs_or_tuple):
+    # Support returning either a plain jobs list or a (jobs, total) tuple
+    if isinstance(jobs_or_tuple, tuple) and len(jobs_or_tuple) == 2:
+        jobs, total = jobs_or_tuple
+    else:
+        jobs = jobs_or_tuple
+        total = None
     for job in jobs:
         job['status'] = job['status'].value
-    return jobs
+    if total is None:
+        return jobs
+    return {'jobs': jobs, 'total': total}
 
 
 def _encode_serve_status(

sky/server/server.py

@@ -792,8 +792,6 @@ async def validate(validate_body: payloads.ValidateBody) -> None:
     ctx.override_envs(validate_body.env_vars)
 
     def validate_dag(dag: dag_utils.dag_lib.Dag):
-        # Resolve the volumes before admin policy and validation.
-        dag.resolve_and_validate_volumes()
         # TODO: Admin policy may contain arbitrary code, which may be expensive
         # to run and may block the server thread. However, moving it into the
         # executor adds a ~150ms penalty on the local API server because of
@@ -802,6 +800,7 @@ async def validate(validate_body: payloads.ValidateBody) -> None:
         with admin_policy_utils.apply_and_use_config_in_current_request(
                 dag,
                 request_options=validate_body.get_request_options()) as dag:
+            dag.resolve_and_validate_volumes()
             # Skip validating workdir and file_mounts, as those need to be
             # validated after the files are uploaded to the SkyPilot API server
             # with `upload_mounts_to_api_server`.
@@ -1284,6 +1283,46 @@ async def download(download_body: payloads.DownloadBody) -> None:
                                     detail=f'Error creating zip file: {str(e)}')
 
 
+@app.post('/provision_logs')
+async def provision_logs(cluster_body: payloads.ClusterNameBody,
+                         follow: bool = True,
+                         tail: int = 0) -> fastapi.responses.StreamingResponse:
+    """Streams the provision.log for the latest launch request of a cluster."""
+    # Prefer clusters table first, then cluster_history as fallback.
+    log_path_str = global_user_state.get_cluster_provision_log_path(
+        cluster_body.cluster_name)
+    if not log_path_str:
+        log_path_str = global_user_state.get_cluster_history_provision_log_path(
+            cluster_body.cluster_name)
+    if not log_path_str:
+        raise fastapi.HTTPException(
+            status_code=404,
+            detail=('Provision log path is not recorded for this cluster. '
+                    'Please relaunch to generate provisioning logs.'))
+
+    log_path = pathlib.Path(log_path_str).expanduser().resolve()
+    if not log_path.exists():
+        raise fastapi.HTTPException(
+            status_code=404,
+            detail=f'Provision log path does not exist: {str(log_path)}')
+
+    # Tail semantics: 0 means print all lines. Convert 0 -> None for streamer.
+    effective_tail = None if tail is None or tail <= 0 else tail
+
+    return fastapi.responses.StreamingResponse(
+        content=stream_utils.log_streamer(None,
+                                          log_path,
+                                          tail=effective_tail,
+                                          follow=follow),
+        media_type='text/plain',
+        headers={
+            'Cache-Control': 'no-cache, no-transform',
+            'X-Accel-Buffering': 'no',
+            'Transfer-Encoding': 'chunked',
+        },
+    )
+
+
 @app.post('/cost_report')
 async def cost_report(request: fastapi.Request,
                       cost_report_body: payloads.CostReportBody) -> None:
@@ -1815,6 +1854,9 @@ if __name__ == '__main__':
             global_tasks.append(background.create_task(metrics_server.serve()))
         global_tasks.append(
             background.create_task(requests_lib.requests_gc_daemon()))
+        global_tasks.append(
+            background.create_task(
+                global_user_state.cluster_event_retention_daemon()))
         threading.Thread(target=background.run_forever, daemon=True).start()
 
         queue_server, workers = executor.start(config)

sky/templates/kubernetes-ray.yml.j2

@@ -378,6 +378,7 @@ available_node_types:
         {% if volume_mounts %}
         securityContext:
           fsGroup: 1000
+          fsGroupChangePolicy: OnRootMismatch
         {% endif %}
 
         # Add node selector if GPU/TPUs are requested:

sky/utils/common_utils.py

@@ -1,6 +1,7 @@
 """Utils shared between all of sky"""
 
 import difflib
+import enum
 import functools
 import getpass
 import hashlib
@@ -55,6 +56,25 @@ _VALID_ENV_VAR_REGEX = '[a-zA-Z_][a-zA-Z0-9_]*'
 logger = sky_logging.init_logger(__name__)
 
 
+class ProcessStatus(enum.Enum):
+    """Process status."""
+
+    # The process is scheduled to run, but not started yet.
+    SCHEDULED = 'SCHEDULED'
+
+    # The process is running
+    RUNNING = 'RUNNING'
+
+    # The process is finished and succeeded
+    SUCCEEDED = 'SUCCEEDED'
+
+    # The process is interrupted
+    INTERRUPTED = 'INTERRUPTED'
+
+    # The process failed
+    FAILED = 'FAILED'
+
+
 @annotations.lru_cache(scope='request')
 def get_usage_run_id() -> str:
     """Returns a unique run id for each 'run'.

sky/utils/controller_utils.py

@@ -1224,13 +1224,26 @@ def _get_launch_parallelism() -> int:
 
 
 def can_provision() -> bool:
-    num_provision = (
-        serve_state.total_number_provisioning_replicas() * SERVE_LAUNCH_RATIO +
-        managed_job_state.get_num_launching_jobs())
-    return num_provision < _get_launch_parallelism()
+    # We always prioritize terminating over provisioning, to save the cost on
+    # idle resources.
+    if serve_state.total_number_scheduled_to_terminate_replicas() > 0:
+        return False
+    return can_terminate()
 
 
 def can_start_new_process() -> bool:
     num_procs = (serve_state.get_num_services() * SERVE_PROC_RATIO +
                  managed_job_state.get_num_alive_jobs())
     return num_procs < _get_job_parallelism()
+
+
+# We limit the number of terminating replicas to the number of CPUs. This is
+# just a temporary solution to avoid overwhelming the controller. After one job
+# controller PR, we should use API server to handle resources management.
+def can_terminate() -> bool:
+    num_terminating = (
+        serve_state.total_number_provisioning_replicas() * SERVE_LAUNCH_RATIO +
+        # Each terminate process will take roughly the same CPUs as job launch.
+        serve_state.total_number_terminating_replicas() +
+        managed_job_state.get_num_launching_jobs())
+    return num_terminating < _get_launch_parallelism()

sky/utils/db/migration_utils.py

@@ -19,7 +19,7 @@ logger = sky_logging.init_logger(__name__)
 DB_INIT_LOCK_TIMEOUT_SECONDS = 10
 
 GLOBAL_USER_STATE_DB_NAME = 'state_db'
-GLOBAL_USER_STATE_VERSION = '005'
+GLOBAL_USER_STATE_VERSION = '006'
 GLOBAL_USER_STATE_LOCK_PATH = '~/.sky/locks/.state_db.lock'
 
 SPOT_JOBS_DB_NAME = 'spot_jobs_db'

sky/utils/log_utils.py

@@ -47,13 +47,16 @@ class RayUpLineProcessor(LineProcessor):
         RUNTIME_SETUP = 1
         PULLING_DOCKER_IMAGES = 2
 
-    def __init__(self, log_path: str):
+    def __init__(self, log_path: str, cluster_name: Optional[str] = None):
         self.log_path = log_path
+        self.cluster_name = cluster_name
 
     def __enter__(self) -> None:
         self.state = self.ProvisionStatus.LAUNCH
         self.status_display = rich_utils.safe_status(
-            ux_utils.spinner_message('Launching', self.log_path))
+            ux_utils.spinner_message('Launching',
+                                     self.log_path,
+                                     cluster_name=self.cluster_name))
         self.status_display.start()
 
     def process_line(self, log_line: str) -> None:
@@ -62,19 +65,25 @@ class RayUpLineProcessor(LineProcessor):
             logger.info('  Head VM is up.')
             self.status_display.update(
                 ux_utils.spinner_message(
-                    'Launching - Preparing SkyPilot runtime', self.log_path))
+                    'Launching - Preparing SkyPilot runtime',
+                    self.log_path,
+                    cluster_name=self.cluster_name))
             self.state = self.ProvisionStatus.RUNTIME_SETUP
         if ('Pulling from' in log_line and
                 self.state == self.ProvisionStatus.RUNTIME_SETUP):
             self.status_display.update(
                 ux_utils.spinner_message(
-                    'Launching - Initializing docker container', self.log_path))
+                    'Launching - Initializing docker container',
+                    self.log_path,
+                    cluster_name=self.cluster_name))
             self.state = self.ProvisionStatus.PULLING_DOCKER_IMAGES
         if ('Status: Downloaded newer image' in log_line and
                 self.state == self.ProvisionStatus.PULLING_DOCKER_IMAGES):
             self.status_display.update(
                 ux_utils.spinner_message(
-                    'Launching - Preparing SkyPilot runtime', self.log_path))
+                    'Launching - Preparing SkyPilot runtime',
+                    self.log_path,
+                    cluster_name=self.cluster_name))
             self.state = self.ProvisionStatus.RUNTIME_SETUP
 
     def __exit__(self, except_type: Optional[Type[BaseException]],

sky/utils/resources_utils.py

@@ -5,7 +5,7 @@ import itertools
 import json
 import math
 import typing
-from typing import Dict, List, Optional, Set, Union
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
 
 from sky import skypilot_config
 from sky.skylet import constants
@@ -435,3 +435,27 @@ def parse_time_minutes(time: str) -> int:
                 continue
 
     raise ValueError(f'Invalid time format: {time}')
+
+
+def normalize_any_of_resources_config(
+        any_of: List[Dict[str, Any]]) -> Tuple[str, ...]:
+    """Normalize a list of any_of resources config to a canonical form.
+
+    Args:
+        any_of: A list of any_of resources config.
+
+    Returns:
+        A normalized tuple representation that can be compared for equality.
+        Two lists with the same resource configurations in different orders
+        will produce the same normalized result.
+    """
+    if not any_of:
+        return tuple()
+
+    # Convert each config to JSON string with sorted keys, then sort the list
+    normalized_configs = [
+        json.dumps(config, sort_keys=True, separators=(',', ':'))
+        for config in any_of
+    ]
+
+    return tuple(sorted(normalized_configs))

sky/utils/schemas.py

@@ -1535,6 +1535,9 @@ def get_config_schema():
             'requests_retention_hours': {
                 'type': 'integer',
             },
+            'cluster_event_retention_hours': {
+                'type': 'number',
+            },
         }
     }
 

sky/utils/ux_utils.py

@@ -26,9 +26,16 @@ BOLD = '\033[1m'
 RESET_BOLD = '\033[0m'
 
 # Log path hint in the spinner during launching
+# (old, kept for backward compatibility)
 _LOG_PATH_HINT = (f'{colorama.Style.DIM}View logs: sky api logs -l '
                   '{log_path}'
                   f'{colorama.Style.RESET_ALL}')
+# Log hint: recommend sky logs --provision <cluster_name>
+_PROVISION_LOG_HINT = (
+    f'{colorama.Style.DIM}View logs: '
+    f'{BOLD}sky logs --provision {{cluster_name}}{RESET_BOLD}'
+    f'{colorama.Style.RESET_ALL}')
+# Legacy path hint retained for local-only cases where we don't have cluster
 _LOG_PATH_HINT_LOCAL = (f'{colorama.Style.DIM}View logs: '
                         '{log_path}'
                         f'{colorama.Style.RESET_ALL}')
@@ -126,7 +133,10 @@ class RedirectOutputForProcess:
 
 def log_path_hint(log_path: Union[str, 'pathlib.Path'],
                   is_local: bool = False) -> str:
-    """Gets the log path hint for the given log path."""
+    """Gets the log path hint for the given log path.
+
+    Kept for backward compatibility when only paths are available.
+    """
     log_path = str(log_path)
     expanded_home = os.path.expanduser('~')
     if log_path.startswith(expanded_home):
@@ -139,6 +149,12 @@ def log_path_hint(log_path: Union[str, 'pathlib.Path'],
     return _LOG_PATH_HINT.format(log_path=log_path)
 
 
+def provision_hint(cluster_name: Optional[str]) -> Optional[str]:
+    if not cluster_name:
+        return None
+    return _PROVISION_LOG_HINT.format(cluster_name=cluster_name)
+
+
 def starting_message(message: str) -> str:
     """Gets the starting message for the given message."""
     # We have to reset the color before the message, because sometimes if a
@@ -150,7 +166,8 @@ def starting_message(message: str) -> str:
 def finishing_message(message: str,
                       log_path: Optional[Union[str, 'pathlib.Path']] = None,
                       is_local: bool = False,
-                      follow_up_message: Optional[str] = None) -> str:
+                      follow_up_message: Optional[str] = None,
+                      cluster_name: Optional[str] = None) -> str:
     """Gets the finishing message for the given message.
 
     Args:
@@ -168,6 +185,9 @@ def finishing_message(message: str,
     success_prefix = (f'{colorama.Style.RESET_ALL}{colorama.Fore.GREEN}✓ '
                       f'{message}{colorama.Style.RESET_ALL}{follow_up_message}'
                       f'{colorama.Style.RESET_ALL}')
+    hint = provision_hint(cluster_name)
+    if hint:
+        return f'{success_prefix}  {hint}'
     if log_path is None:
         return success_prefix
     path_hint = log_path_hint(log_path, is_local)
@@ -176,13 +196,17 @@ def finishing_message(message: str,
 
 def error_message(message: str,
                   log_path: Optional[Union[str, 'pathlib.Path']] = None,
-                  is_local: bool = False) -> str:
+                  is_local: bool = False,
+                  cluster_name: Optional[str] = None) -> str:
     """Gets the error message for the given message."""
     # We have to reset the color before the message, because sometimes if a
     # previous spinner with dimmed color overflows in a narrow terminal, the
     # color might be messed up.
     error_prefix = (f'{colorama.Style.RESET_ALL}{colorama.Fore.RED}⨯'
                     f'{colorama.Style.RESET_ALL} {message}')
+    hint = provision_hint(cluster_name)
+    if hint:
+        return f'{error_prefix}  {hint}'
     if log_path is None:
         return error_prefix
     path_hint = log_path_hint(log_path, is_local)
@@ -200,9 +224,16 @@ def retry_message(message: str) -> str:
 
 def spinner_message(message: str,
                     log_path: Optional[Union[str, 'pathlib.Path']] = None,
-                    is_local: bool = False) -> str:
-    """Gets the spinner message for the given message and log path."""
+                    is_local: bool = False,
+                    cluster_name: Optional[str] = None) -> str:
+    """Gets the spinner message for the given message and log path.
+
+    If cluster_name is provided, recommend `sky logs --provision <cluster>`.
+    """
     colored_spinner = f'[bold cyan]{message}[/]'
+    hint = provision_hint(cluster_name)
+    if hint:
+        return f'{colored_spinner}  {hint}'
     if log_path is None:
         return colored_spinner
     path_hint = log_path_hint(log_path, is_local)

{skypilot_nightly-1.0.0.dev20250814.dist-info → skypilot_nightly-1.0.0.dev20250815.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skypilot-nightly
-Version: 1.0.0.dev20250814
+Version: 1.0.0.dev20250815
 Summary: SkyPilot: Run AI on Any Infra — Unified, Faster, Cheaper.
 Author: SkyPilot Team
 License: Apache 2.0