skypilot-nightly 1.0.0.dev20250804__py3-none-any.whl → 1.0.0.dev20250807__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '641a8e762a1d86274420f7a9d9cea6657b5be5b4'
+_SKYPILOT_COMMIT_SHA = 'a167cba8230b0ffda6baa0c825fa0eb5d5ab4aa4'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250804'
+__version__ = '1.0.0.dev20250807'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

sky/backends/cloud_vm_ray_backend.py

@@ -168,6 +168,9 @@ _MAX_INLINE_SCRIPT_LENGTH = 100 * 1024
 _RESOURCES_UNAVAILABLE_LOG = (
     'Reasons for provision failures (for details, please check the log above):')
 
+# Number of seconds to wait locking the cluster before communicating with user.
+_CLUSTER_LOCK_TIMEOUT = 5.0
+
 
 def _is_command_length_over_limit(command: str) -> bool:
     """Check if the length of the command exceeds the limit.
@@ -2917,10 +2920,36 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
         # exceptions.ClusterOwnerIdentityMismatchError
         backend_utils.check_owner_identity(cluster_name)
         lock_id = backend_utils.cluster_status_lock_id(cluster_name)
-        with timeline.DistributedLockEvent(lock_id):
-            # Try to launch the exiting cluster first. If no existing cluster,
-            # this function will create a to_provision_config with required
-            # resources.
+        communicated_with_user = False
+
+        while True:
+            try:
+                return self._locked_provision(lock_id, task, to_provision,
+                                              dryrun, stream_logs, cluster_name,
+                                              retry_until_up,
+                                              skip_unnecessary_provisioning)
+            except locks.LockTimeout:
+                if not communicated_with_user:
+                    logger.info(f'{colorama.Fore.YELLOW}'
+                                f'Launching delayed, check concurrent tasks: '
+                                f'sky api status')
+                    communicated_with_user = True
+
+    def _locked_provision(
+        self,
+        lock_id: str,
+        task: task_lib.Task,
+        to_provision: Optional[resources_lib.Resources],
+        dryrun: bool,
+        stream_logs: bool,
+        cluster_name: str,
+        retry_until_up: bool = False,
+        skip_unnecessary_provisioning: bool = False,
+    ) -> Tuple[Optional[CloudVmRayResourceHandle], bool]:
+        with timeline.DistributedLockEvent(lock_id, _CLUSTER_LOCK_TIMEOUT):
+            # Try to launch the exiting cluster first. If no existing
+            # cluster, this function will create a to_provision_config
+            # with required resources.
             to_provision_config = self._check_existing_cluster(
                 task, to_provision, cluster_name, dryrun)
             assert to_provision_config.resources is not None, (

sky/catalog/kubernetes_catalog.py

@@ -255,6 +255,14 @@ def _list_accelerators(
                     # Get all the pods running on the node
                     if (pod.spec.node_name == node.metadata.name and
                             pod.status.phase in ['Running', 'Pending']):
+                        # Skip pods that should not count against GPU count
+                        if (kubernetes_utils.
+                                should_exclude_pod_from_gpu_allocation(pod)):
+                            logger.debug(
+                                f'Excluding pod '
+                                f'{pod.metadata.name} from GPU count '
+                                f'calculations on node {node.metadata.name}')
+                            continue
                         # Iterate over all the containers in the pod and sum
                         # the GPU requests
                         for container in pod.spec.containers:

sky/catalog/nebius_catalog.py

@@ -38,7 +38,6 @@ def get_hourly_cost(instance_type: str,
                     region: Optional[str] = None,
                     zone: Optional[str] = None) -> float:
     """Returns the cost, or the cheapest cost among all zones for spot."""
-    assert not use_spot, 'Nebius does not support spot.'
     if zone is not None:
         with ux_utils.print_exception_no_traceback():
             raise ValueError('Nebius does not support zones.')

sky/check.py

@@ -467,8 +467,18 @@ def _print_checked_cloud(
         if ok:
             enabled_capabilities.append(capability)
         # `dict` reasons for K8s and SSH will be printed in detail in
-        # _format_enabled_cloud. Skip here.
+        # _format_enabled_cloud. Skip here unless the cloud is disabled.
         if not isinstance(reason, str):
+            if not ok and isinstance(cloud_tuple[1],
+                                     (sky_clouds.SSH, sky_clouds.Kubernetes)):
+                if reason is not None:
+                    reason_str = _format_context_details(cloud_tuple[1],
+                                                         show_details=True,
+                                                         ctx2text=reason)
+                    reason_str = '\n'.join(
+                        '    ' + line for line in reason_str.splitlines())
+                    reasons_to_capabilities.setdefault(reason_str,
+                                                       []).append(capability)
             continue
         if ok:
             if reason is not None:

sky/client/cli/command.py

@@ -556,17 +556,17 @@ def _parse_override_params(
     """Parses the override parameters into a dictionary."""
     override_params: Dict[str, Any] = {}
     if cloud is not None:
-        if cloud.lower() == 'none':
+        if cloud.lower() == 'none' or cloud == '*':
             override_params['cloud'] = None
         else:
             override_params['cloud'] = registry.CLOUD_REGISTRY.from_str(cloud)
     if region is not None:
-        if region.lower() == 'none':
+        if region.lower() == 'none' or region == '*':
             override_params['region'] = None
         else:
             override_params['region'] = region
     if zone is not None:
-        if zone.lower() == 'none':
+        if zone.lower() == 'none' or zone == '*':
             override_params['zone'] = None
         else:
             override_params['zone'] = zone
@@ -964,9 +964,10 @@ def _handle_infra_cloud_region_zone_options(infra: Optional[str],
 
     if infra is not None:
         infra_info = infra_utils.InfraInfo.from_str(infra)
-        cloud = infra_info.cloud
-        region = infra_info.region
-        zone = infra_info.zone
+        # Convert None to '*' to ensure proper override behavior
+        cloud = infra_info.cloud if infra_info.cloud is not None else '*'
+        region = infra_info.region if infra_info.region is not None else '*'
+        zone = infra_info.zone if infra_info.zone is not None else '*'
     return cloud, region, zone
 
 
@@ -3462,6 +3463,17 @@ def show_gpus(
                                                                region,
                                                                zone=None)
 
+    # cloud and region could be '*' from _handle_infra_cloud_region_zone_options
+    # which normally indicates to
+    # _make_task_or_dag_from_entrypoint_with_overrides -> _parse_override_params
+    # to disregard the cloud and region from the YAML.
+    # In show_gpus, there is no YAML, so we need to handle the '*' value
+    # directly here. We should use None instead to indicate "any".
+    if cloud == '*':
+        cloud = None
+    if region == '*':
+        region = None
+
     # validation for the --region flag
     if region is not None and cloud is None:
         raise click.UsageError(
@@ -3502,7 +3514,14 @@ def show_gpus(
                               (cloud_name is None or cloud_is_ssh))
 
     def _list_to_str(lst):
-        return ', '.join([str(e) for e in lst])
+
+        def format_number(n):
+            # If it's a float that's a whole number, display as int
+            if isinstance(n, float) and n.is_integer():
+                return str(int(n))
+            return str(n)
+
+        return ', '.join([format_number(n) for n in lst])
 
     # TODO(zhwu,romilb): We should move most of these kubernetes related
     # queries into the backend, especially behind the server.
@@ -4953,6 +4972,205 @@ def jobs_pool_down(
     _async_call_or_wait(request_id, async_call, 'sky.jobs.pool_down')
 
 
+def _handle_serve_logs(
+        service_name: str,
+        follow: bool,
+        controller: bool,
+        load_balancer: bool,
+        replica_ids: Tuple[int, ...],
+        sync_down: bool,
+        tail: Optional[int],
+        pool: bool,  # pylint: disable=redefined-outer-name
+):
+    noun = 'pool' if pool else 'service'
+    capnoun = noun.capitalize()
+    repnoun = 'worker' if pool else 'replica'
+    if tail is not None:
+        if tail < 0:
+            raise click.UsageError('--tail must be a non-negative integer.')
+        # TODO(arda): We could add ability to tail and follow logs together.
+        if follow:
+            follow = False
+            logger.warning(
+                f'{colorama.Fore.YELLOW}'
+                '--tail and --follow cannot be used together. '
+                f'Changed the mode to --no-follow.{colorama.Style.RESET_ALL}')
+
+    chosen_components: Set[serve_lib.ServiceComponent] = set()
+    if controller:
+        chosen_components.add(serve_lib.ServiceComponent.CONTROLLER)
+    if load_balancer:
+        assert not pool, 'Load balancer is not supported for pools.'
+        chosen_components.add(serve_lib.ServiceComponent.LOAD_BALANCER)
+    # replica_ids contains the specific replica IDs provided by the user.
+    # If it's not empty, it implies the user wants replica logs.
+    if replica_ids:
+        chosen_components.add(serve_lib.ServiceComponent.REPLICA)
+
+    if sync_down:
+        # For sync-down, multiple targets are allowed.
+        # If no specific components/replicas are mentioned, sync all.
+        # Note: Multiple replicas or targets can only be specified when
+        # using --sync-down.
+        targets_to_sync = list(chosen_components)
+        if not targets_to_sync and not replica_ids:
+            # Default to all components if nothing specific is requested
+            targets_to_sync = [
+                serve_lib.ServiceComponent.CONTROLLER,
+                serve_lib.ServiceComponent.REPLICA,
+            ]
+            if not pool:
+                targets_to_sync.append(serve_lib.ServiceComponent.LOAD_BALANCER)
+
+        timestamp = sky_logging.get_run_timestamp()
+        log_dir = (pathlib.Path(constants.SKY_LOGS_DIRECTORY) / noun /
+                   f'{service_name}_{timestamp}').expanduser()
+        log_dir.mkdir(parents=True, exist_ok=True)
+
+        with rich_utils.client_status(
+                ux_utils.spinner_message(f'Downloading {noun} logs...')):
+            if pool:
+                managed_jobs.pool_sync_down_logs(service_name,
+                                                 str(log_dir),
+                                                 targets=targets_to_sync,
+                                                 worker_ids=list(replica_ids),
+                                                 tail=tail)
+            else:
+                serve_lib.sync_down_logs(service_name,
+                                         str(log_dir),
+                                         targets=targets_to_sync,
+                                         replica_ids=list(replica_ids),
+                                         tail=tail)
+        style = colorama.Style
+        fore = colorama.Fore
+        logger.info(f'{fore.CYAN}{capnoun} {service_name} logs: '
+                    f'{log_dir}{style.RESET_ALL}')
+        return
+
+    # Tailing requires exactly one target.
+    num_targets = len(chosen_components)
+    # If REPLICA component is chosen, len(replica_ids) must be 1 for tailing.
+    if serve_lib.ServiceComponent.REPLICA in chosen_components:
+        if len(replica_ids) != 1:
+            raise click.UsageError(
+                f'Can only tail logs from a single {repnoun} at a time. '
+                f'Provide exactly one {repnoun.upper()}_ID or use --sync-down '
+                f'to download logs from multiple {repnoun}s.')
+        # If replica is chosen and len is 1, num_targets effectively counts it.
+        # We need to ensure no other component (controller/LB) is selected.
+        if num_targets > 1:
+            raise click.UsageError(
+                'Can only tail logs from one target at a time (controller, '
+                f'load balancer, or a single {repnoun}). Use --sync-down '
+                'to download logs from multiple sources.')
+    elif num_targets == 0:
+        raise click.UsageError(
+            'Specify a target to tail: --controller, --load-balancer, or '
+            f'a {repnoun.upper()}_ID.')
+    elif num_targets > 1:
+        raise click.UsageError(
+            'Can only tail logs from one target at a time. Use --sync-down '
+            'to download logs from multiple sources.')
+
+    # At this point, we have exactly one target for tailing.
+    assert len(chosen_components) == 1
+    assert len(replica_ids) in [0, 1]
+    target_component = chosen_components.pop()
+    target_replica_id: Optional[int] = replica_ids[0] if replica_ids else None
+
+    try:
+        if pool:
+            managed_jobs.pool_tail_logs(service_name,
+                                        target=target_component,
+                                        worker_id=target_replica_id,
+                                        follow=follow,
+                                        tail=tail)
+        else:
+            serve_lib.tail_logs(service_name,
+                                target=target_component,
+                                replica_id=target_replica_id,
+                                follow=follow,
+                                tail=tail)
+    except exceptions.ClusterNotUpError:
+        with ux_utils.print_exception_no_traceback():
+            raise
+
+
+@pool.command('logs', cls=_DocumentedCodeCommand)
+@flags.config_option(expose_value=False)
+@click.option(
+    '--follow/--no-follow',
+    is_flag=True,
+    default=True,
+    help=('Follow the logs of the job. [default: --follow] '
+          'If --no-follow is specified, print the log so far and exit.'))
+@click.option('--controller',
+              is_flag=True,
+              default=False,
+              required=False,
+              help='Show the controller logs of this pool.')
+@click.option('--sync-down',
+              '-s',
+              is_flag=True,
+              default=False,
+              help='Sync down logs to the local machine. Can be combined with '
+              '--controller or worker ID to narrow scope.')
+@click.option(
+    '--tail',
+    default=None,
+    type=int,
+    help='The number of lines to display from the end of the log file. '
+    'Default is None, which means print all lines.')
+@click.argument('pool_name', required=True, type=str)
+@click.argument('worker_ids', required=False, type=int, nargs=-1)
+@usage_lib.entrypoint
+# TODO(tian): Add default argument for this CLI if none of the flags are
+# specified.
+def pool_logs(
+    pool_name: str,
+    follow: bool,
+    controller: bool,
+    worker_ids: Tuple[int, ...],
+    sync_down: bool,
+    tail: Optional[int],
+):
+    """Tail or sync down logs of a pool.
+
+    Logs can be tailed from one target (controller, or a single worker) or
+    synced down from multiple targets simultaneously.
+
+    Example:
+
+    .. code-block:: bash
+
+        # Tail the controller logs of a pool
+        sky pool logs --controller [POOL_NAME]
+        \b
+        # Print the worker logs so far and exit
+        sky pool logs --no-follow [POOL_NAME]
+        \b
+        # Tail the logs of worker 1
+        sky pool logs [POOL_NAME] 1
+        \b
+        # Show the last 100 lines of the controller logs
+        sky pool logs --controller --tail 100 [POOL_NAME]
+        \b
+        # Sync down all logs of the pool (controller, all workers)
+        sky pool logs [POOL_NAME] --sync-down
+        \b
+        # Sync down controller logs and logs for workers 1 and 3
+        sky pool logs [POOL_NAME] 1 3 --controller --sync-down
+    """
+    _handle_serve_logs(pool_name,
+                       follow=follow,
+                       controller=controller,
+                       load_balancer=False,
+                       replica_ids=worker_ids,
+                       sync_down=sync_down,
+                       tail=tail,
+                       pool=True)
+
+
 @cli.command(cls=_DocumentedCodeCommand)
 @flags.config_option(expose_value=False)
 @usage_lib.entrypoint
@@ -5536,6 +5754,7 @@ def serve_down(
                           show_default=True)
 
     if replica_id_is_defined:
+        assert replica_id is not None
         request_id = serve_lib.terminate_replica(service_names[0], replica_id,
                                                  purge)
     else:
@@ -5616,99 +5835,14 @@ def serve_logs(
         # Sync down controller logs and logs for replicas 1 and 3
         sky serve logs [SERVICE_NAME] 1 3 --controller --sync-down
     """
-    if tail is not None:
-        if tail < 0:
-            raise click.UsageError('--tail must be a non-negative integer.')
-        # TODO(arda): We could add ability to tail and follow logs together.
-        if follow:
-            follow = False
-            logger.warning(
-                f'{colorama.Fore.YELLOW}'
-                '--tail and --follow cannot be used together. '
-                f'Changed the mode to --no-follow.{colorama.Style.RESET_ALL}')
-
-    chosen_components: Set[serve_lib.ServiceComponent] = set()
-    if controller:
-        chosen_components.add(serve_lib.ServiceComponent.CONTROLLER)
-    if load_balancer:
-        chosen_components.add(serve_lib.ServiceComponent.LOAD_BALANCER)
-    # replica_ids contains the specific replica IDs provided by the user.
-    # If it's not empty, it implies the user wants replica logs.
-    if replica_ids:
-        chosen_components.add(serve_lib.ServiceComponent.REPLICA)
-
-    if sync_down:
-        # For sync-down, multiple targets are allowed.
-        # If no specific components/replicas are mentioned, sync all.
-        # Note: Multiple replicas or targets can only be specified when
-        # using --sync-down.
-        targets_to_sync = list(chosen_components)
-        if not targets_to_sync and not replica_ids:
-            # Default to all components if nothing specific is requested
-            targets_to_sync = [
-                serve_lib.ServiceComponent.CONTROLLER,
-                serve_lib.ServiceComponent.LOAD_BALANCER,
-                serve_lib.ServiceComponent.REPLICA,
-            ]
-
-        timestamp = sky_logging.get_run_timestamp()
-        log_dir = (pathlib.Path(constants.SKY_LOGS_DIRECTORY) / 'service' /
-                   f'{service_name}_{timestamp}').expanduser()
-        log_dir.mkdir(parents=True, exist_ok=True)
-
-        with rich_utils.client_status(
-                ux_utils.spinner_message('Downloading service logs...')):
-            serve_lib.sync_down_logs(service_name,
-                                     local_dir=str(log_dir),
-                                     targets=targets_to_sync,
-                                     replica_ids=list(replica_ids),
-                                     tail=tail)
-        style = colorama.Style
-        fore = colorama.Fore
-        logger.info(f'{fore.CYAN}Service {service_name} logs: '
-                    f'{log_dir}{style.RESET_ALL}')
-        return
-
-    # Tailing requires exactly one target.
-    num_targets = len(chosen_components)
-    # If REPLICA component is chosen, len(replica_ids) must be 1 for tailing.
-    if serve_lib.ServiceComponent.REPLICA in chosen_components:
-        if len(replica_ids) != 1:
-            raise click.UsageError(
-                'Can only tail logs from a single replica at a time. '
-                'Provide exactly one REPLICA_ID or use --sync-down '
-                'to download logs from multiple replicas.')
-        # If replica is chosen and len is 1, num_targets effectively counts it.
-        # We need to ensure no other component (controller/LB) is selected.
-        if num_targets > 1:
-            raise click.UsageError(
-                'Can only tail logs from one target at a time (controller, '
-                'load balancer, or a single replica). Use --sync-down '
-                'to download logs from multiple sources.')
-    elif num_targets == 0:
-        raise click.UsageError(
-            'Specify a target to tail: --controller, --load-balancer, or '
-            'a REPLICA_ID.')
-    elif num_targets > 1:
-        raise click.UsageError(
-            'Can only tail logs from one target at a time. Use --sync-down '
-            'to download logs from multiple sources.')
-
-    # At this point, we have exactly one target for tailing.
-    assert len(chosen_components) == 1
-    assert len(replica_ids) in [0, 1]
-    target_component = chosen_components.pop()
-    target_replica_id: Optional[int] = replica_ids[0] if replica_ids else None
-
-    try:
-        serve_lib.tail_logs(service_name,
-                            target=target_component,
-                            replica_id=target_replica_id,
-                            follow=follow,
-                            tail=tail)
-    except exceptions.ClusterNotUpError:
-        with ux_utils.print_exception_no_traceback():
-            raise
+    _handle_serve_logs(service_name,
+                       follow=follow,
+                       controller=controller,
+                       load_balancer=load_balancer,
+                       replica_ids=replica_ids,
+                       sync_down=sync_down,
+                       tail=tail,
+                       pool=False)
 
 
 @cli.group(cls=_NaturalOrderGroup, hidden=True)

sky/client/sdk.py

@@ -88,14 +88,17 @@ def reload_config() -> None:
     skypilot_config.safe_reload_config()
 
 
-def stream_response(request_id: Optional[str],
+def stream_response(request_id: Optional[server_common.RequestId],
                     response: 'requests.Response',
                     output_stream: Optional['io.TextIOBase'] = None,
                     resumable: bool = False) -> Any:
     """Streams the response to the console.
 
     Args:
-        request_id: The request ID.
+        request_id: The request ID of the request to stream. May be a full
+            request ID or a prefix.
+            If None, the latest request submitted to the API server is streamed.
+            Using None request_id is not recommended in multi-user environments.
         response: The HTTP response.
         output_stream: The output stream to write to. If None, print to the
             console.
@@ -349,7 +352,14 @@ def validate(
             validation. This is only required when a admin policy is in use,
             see: https://docs.skypilot.co/en/latest/cloud-setup/policy.html
     """
+    remote_api_version = versions.get_remote_api_version()
+    # TODO(kevin): remove this in v0.13.0
+    omit_user_specified_yaml = (remote_api_version is None or
+                                remote_api_version < 15)
     for task in dag.tasks:
+        if omit_user_specified_yaml:
+            # pylint: disable=protected-access
+            task._user_specified_yaml = None
         task.expand_and_validate_workdir()
         if not workdir_only:
             task.expand_and_validate_file_mounts()
@@ -1756,7 +1766,7 @@ def status_kubernetes() -> server_common.RequestId:
 # === API request APIs ===
 @usage_lib.entrypoint
 @annotations.client_api
-def get(request_id: str) -> Any:
+def get(request_id: server_common.RequestId) -> Any:
     """Waits for and gets the result of a request.
 
     This function will not check the server health since /api/get is typically
@@ -1764,7 +1774,8 @@ def get(request_id: str) -> Any:
     may cause GET /api/get being sent to a restarted API server.
 
     Args:
-        request_id: The request ID of the request to get.
+        request_id: The request ID of the request to get. May be a full request
+            ID or a prefix.
 
     Returns:
         The ``Request Returns`` of the specified request. See the documentation
@@ -1818,7 +1829,7 @@ def get(request_id: str) -> Any:
 @server_common.check_server_healthy_or_start
 @annotations.client_api
 def stream_and_get(
-    request_id: Optional[str] = None,
+    request_id: Optional[server_common.RequestId] = None,
     log_path: Optional[str] = None,
     tail: Optional[int] = None,
     follow: bool = True,
@@ -1830,7 +1841,10 @@ def stream_and_get(
     prefix of the full request id.
 
     Args:
-        request_id: The prefix of the request ID of the request to stream.
+        request_id: The request ID of the request to stream. May be a full
+            request ID or a prefix.
+            If None, the latest request submitted to the API server is streamed.
+            Using None request_id is not recommended in multi-user environments.
         log_path: The path to the log file to stream.
         tail: The number of lines to show from the end of the logs.
             If None, show all logs.
@@ -1867,13 +1881,18 @@ def stream_and_get(
         with ux_utils.print_exception_no_traceback():
             raise RuntimeError(f'Failed to stream logs: {detail}')
     elif response.status_code != 200:
+        # TODO(syang): handle the case where the requestID is not provided
+        # see https://github.com/skypilot-org/skypilot/issues/6549
+        if request_id is None:
+            return None
         return get(request_id)
     return stream_response(request_id, response, output_stream)
 
 
 @usage_lib.entrypoint
 @annotations.client_api
-def api_cancel(request_ids: Optional[Union[str, List[str]]] = None,
+def api_cancel(request_ids: Optional[Union[
+    server_common.RequestId, List[server_common.RequestId]]] = None,
                all_users: bool = False,
                silent: bool = False) -> server_common.RequestId:
     """Aborts a request or all requests.
@@ -1938,7 +1957,7 @@ def _local_api_server_running(kill: bool = False) -> bool:
 @usage_lib.entrypoint
 @annotations.client_api
 def api_status(
-    request_ids: Optional[List[str]] = None,
+    request_ids: Optional[List[server_common.RequestId]] = None,
     # pylint: disable=redefined-builtin
     all_status: bool = False
 ) -> List[payloads.RequestPayload]:
@@ -2412,7 +2431,9 @@ def api_login(endpoint: Optional[str] = None,
     _save_config_updates(endpoint=endpoint)
     dashboard_url = server_common.get_dashboard_url(endpoint)
 
-    server_common.get_api_server_status.cache_clear()
+    # see https://github.com/python/mypy/issues/5107 on why
+    # typing is disabled on this line
+    server_common.get_api_server_status.cache_clear()  # type: ignore
     # After successful authentication, check server health again to get user
     # identity
     server_status, final_api_server_info = server_common.check_server_healthy(