skypilot-nightly 1.0.0.dev20250521__py3-none-any.whl → 1.0.0.dev20250523__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = 'e1716e31bee6a918b246f8134f2e086e7011bdc8'
+_SKYPILOT_COMMIT_SHA = '8848c5c597089bae431284150803f4e557383b9e'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250521'
+__version__ = '1.0.0.dev20250523'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

sky/adaptors/kubernetes.py

@@ -3,8 +3,8 @@ import logging
 import os
 from typing import Any, Callable, Optional, Set
 
+from sky import sky_logging
 from sky.adaptors import common
-from sky.sky_logging import set_logging_level
 from sky.utils import annotations
 from sky.utils import common_utils
 from sky.utils import ux_utils
@@ -26,6 +26,8 @@ DEFAULT_IN_CLUSTER_REGION = 'in-cluster'
 # set to DEFAULT_IN_CLUSTER_REGION.
 IN_CLUSTER_CONTEXT_NAME_ENV_VAR = 'SKYPILOT_IN_CLUSTER_CONTEXT_NAME'
 
+logger = sky_logging.init_logger(__name__)
+
 
 def _decorate_methods(obj: Any, decorator: Callable, decoration_type: str):
     for attr_name in dir(obj):
@@ -43,7 +45,7 @@ def _decorate_methods(obj: Any, decorator: Callable, decoration_type: str):
     return obj
 
 
-def _api_logging_decorator(logger: str, level: int):
+def _api_logging_decorator(logger_src: str, level: int):
     """Decorator to set logging level for API calls.
 
     This is used to suppress the verbose logging from urllib3 when calls to the
@@ -54,7 +56,9 @@ def _api_logging_decorator(logger: str, level: int):
 
         def wrapped(*args, **kwargs):
             obj = api(*args, **kwargs)
-            _decorate_methods(obj, set_logging_level(logger, level), 'api_log')
+            _decorate_methods(obj,
+                              sky_logging.set_logging_level(logger_src, level),
+                              'api_log')
             return obj
 
         return wrapped
@@ -71,27 +75,53 @@ def _load_config(context: Optional[str] = None):
         except kubernetes.config.config_exception.ConfigException as e:
             suffix = common_utils.format_exception(e, use_bracket=True)
             context_name = '(current-context)' if context is None else context
+            is_ssh_node_pool = False
+            if context_name.startswith('ssh-'):
+                context_name = context_name.lstrip('ssh-')
+                is_ssh_node_pool = True
             # Check if exception was due to no current-context
             if 'Expected key current-context' in str(e):
-                err_str = ('Failed to load Kubernetes configuration for '
-                           f'{context_name!r}. '
-                           'Kubeconfig does not contain any valid context(s).'
-                           f'\n{suffix}\n'
-                           '    If you were running a local Kubernetes '
-                           'cluster, run `sky local up` to start the cluster.')
+                if is_ssh_node_pool:
+                    context_name = context_name.lstrip('ssh-')
+                    err_str = ('Failed to load SSH Node Pool configuration for '
+                               f'{context_name!r}.\n'
+                               '    Run `sky ssh up --infra {context_name}` to '
+                               'set up or repair the cluster.')
+                else:
+                    err_str = (
+                        'Failed to load Kubernetes configuration for '
+                        f'{context_name!r}. '
+                        'Kubeconfig does not contain any valid context(s).'
+                        f'\n{suffix}\n'
+                        '    If you were running a local Kubernetes '
+                        'cluster, run `sky local up` to start the cluster.')
             else:
                 kubeconfig_path = os.environ.get('KUBECONFIG', '~/.kube/config')
-                err_str = (
-                    f'Failed to load Kubernetes configuration for '
-                    f'{context_name!r}. Please check if your kubeconfig file '
-                    f'exists at {kubeconfig_path} and is valid.\n{suffix}')
-            err_str += '\nTo disable Kubernetes for SkyPilot: run `sky check`.'
-            if context is None:  # kubernetes defaults to current-context.
+                if is_ssh_node_pool:
+                    err_str = (
+                        f'Failed to load SSH Node Pool configuration for '
+                        f'{context_name!r}. Run `sky ssh up --infra '
+                        f'{context_name}` to set up or repair the cluster.')
+                else:
+                    err_str = (
+                        'Failed to load Kubernetes configuration for '
+                        f'{context_name!r}. Please check if your kubeconfig '
+                        f'file exists at {kubeconfig_path} and is valid.'
+                        f'\n{suffix}\n')
+            if is_ssh_node_pool:
+                err_str += (f'\nTo disable SSH Node Pool {context_name!r}: '
+                            'run `sky check`.')
+            else:
                 err_str += (
                     '\nHint: Kubernetes attempted to query the current-context '
                     'set in kubeconfig. Check if the current-context is valid.')
             with ux_utils.print_exception_no_traceback():
-                raise ValueError(err_str) from None
+                if is_ssh_node_pool:
+                    # For SSH Node Pool, we don't want to surface k8s errors
+                    # (e.g., missing context) unless debug flag is set.
+                    logging.debug(f'Kubernetes error: {suffix}')
+                else:
+                    raise ValueError(err_str) from None
 
     if context == in_cluster_context_name() or context is None:
         try:

sky/backends/cloud_vm_ray_backend.py

@@ -192,6 +192,7 @@ def _get_cluster_config_template(cloud):
         clouds.DO: 'do-ray.yml.j2',
         clouds.RunPod: 'runpod-ray.yml.j2',
         clouds.Kubernetes: 'kubernetes-ray.yml.j2',
+        clouds.SSH: 'kubernetes-ray.yml.j2',
         clouds.Vsphere: 'vsphere-ray.yml.j2',
         clouds.Vast: 'vast-ray.yml.j2',
         clouds.Fluidstack: 'fluidstack-ray.yml.j2',
@@ -1547,11 +1548,13 @@ class RetryingVmProvisioner(object):
                     controller_str = ('' if controller is None else
                                       f' {controller.value.name}')
                     if isinstance(to_provision.cloud, clouds.Kubernetes):
-                        # Omit the region name for Kubernetes.
+                        suffix = '.'
+                        if region.name.startswith('ssh-'):
+                            suffix = f' ({region.name.lstrip("ssh-")})'
                         logger.info(
                             ux_utils.starting_message(
                                 f'Launching{controller_str} on '
-                                f'{to_provision.cloud}.'))
+                                f'{to_provision.cloud}{suffix}'))
                     else:
                         logger.info(
                             ux_utils.starting_message(
@@ -1724,8 +1727,17 @@ class RetryingVmProvisioner(object):
                 f'{requested_resources}. ')
         elif to_provision.region is not None:
             # For public clouds, provision.region is always set.
-            message = ('Failed to acquire resources in all zones in '
-                       f'{to_provision.region} for {requested_resources}. ')
+            if to_provision.cloud.is_same_cloud(clouds.SSH()):
+                message = ('Failed to acquire resources in SSH Node Pool '
+                           f'({to_provision.region.lstrip("ssh-")}) for '
+                           f'{requested_resources}. The SSH Node Pool may not '
+                           'have enough resources.')
+            elif to_provision.cloud.is_same_cloud(clouds.Kubernetes()):
+                message = ('Failed to acquire resources in context '
+                           f'{to_provision.region} for {requested_resources}. ')
+            else:
+                message = ('Failed to acquire resources in all zones in '
+                           f'{to_provision.region} for {requested_resources}. ')
         else:
             message = (f'Failed to acquire resources in {to_provision.cloud} '
                        f'for {requested_resources}. ')

sky/check.py

@@ -4,8 +4,7 @@ import itertools
 import os
 import traceback
 from types import ModuleType
-from typing import (Any, Callable, Dict, Iterable, List, Optional, Set, Tuple,
-                    Union)
+from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, Union
 
 import click
 import colorama
@@ -44,7 +43,8 @@ def check_capabilities(
     def check_one_cloud_one_capability(
         payload: Tuple[Tuple[str, Union[sky_clouds.Cloud, ModuleType]],
                        sky_cloud.CloudCapability]
-    ) -> Optional[Tuple[sky_cloud.CloudCapability, bool, Optional[str]]]:
+    ) -> Optional[Tuple[sky_cloud.CloudCapability, bool, Optional[Union[
+            str, Dict[str, str]]]]]:
         cloud_tuple, capability = payload
         _, cloud = cloud_tuple
         try:
@@ -53,7 +53,9 @@ def check_capabilities(
             return None
         except Exception:  # pylint: disable=broad-except
             ok, reason = False, traceback.format_exc()
-        return capability, ok, reason.strip() if reason else None
+        if not isinstance(reason, dict):
+            reason = reason.strip() if reason else None
+        return (capability, ok, reason)
 
     def get_cloud_tuple(
             cloud_name: str) -> Tuple[str, Union[sky_clouds.Cloud, ModuleType]]:
@@ -100,13 +102,18 @@ def check_capabilities(
     check_results_dict: Dict[
         Tuple[str, Union[sky_clouds.Cloud, ModuleType]],
         List[Tuple[sky_cloud.CloudCapability, bool,
-                   Optional[str]]]] = collections.defaultdict(list)
+                   Optional[Union[str,
+                                  Dict[str,
+                                       str]]]]]] = collections.defaultdict(list)
+    cloud2ctx2text: Dict[str, Dict[str, str]] = {}
     for combination, check_result in zip(combinations, check_results):
         if check_result is None:
             continue
-        capability, ok, _ = check_result
+        capability, ok, ctx2text = check_result
         cloud_tuple, _ = combination
         cloud_repr = cloud_tuple[0]
+        if isinstance(ctx2text, dict):
+            cloud2ctx2text[cloud_repr] = ctx2text
         if ok:
             enabled_clouds.setdefault(cloud_repr, []).append(capability)
         else:
@@ -115,7 +122,8 @@ def check_capabilities(
 
     for cloud_tuple, check_result_list in sorted(check_results_dict.items(),
                                                  key=lambda item: item[0][0]):
-        _print_checked_cloud(echo, verbose, cloud_tuple, check_result_list)
+        _print_checked_cloud(echo, verbose, cloud_tuple, check_result_list,
+                             cloud2ctx2text.get(cloud_tuple[0], {}))
 
     # Determine the set of enabled clouds: (previously enabled clouds + newly
     # enabled clouds - newly disabled clouds) intersected with
@@ -184,7 +192,8 @@ def check_capabilities(
         # Pretty print for UX.
         if not quiet:
             enabled_clouds_str = '\n  ' + '\n  '.join([
-                _format_enabled_cloud(cloud, capabilities)
+                _format_enabled_cloud(cloud, capabilities,
+                                      cloud2ctx2text.get(cloud, None))
                 for cloud, capabilities in sorted(enabled_clouds.items(),
                                                   key=lambda item: item[0])
             ])
@@ -290,7 +299,8 @@ def _print_checked_cloud(
     verbose: bool,
     cloud_tuple: Tuple[str, Union[sky_clouds.Cloud, ModuleType]],
     cloud_capabilities: List[Tuple[sky_cloud.CloudCapability, bool,
-                                   Optional[str]]],
+                                   Optional[Union[str, Dict[str, str]]]]],
+    ctx2text: Dict[str, str],
 ) -> None:
     """Prints whether a cloud is enabled, and the capabilities that are enabled.
     If any hints (for enabled capabilities) or
@@ -317,24 +327,34 @@ def _print_checked_cloud(
     for capability, ok, reason in cloud_capabilities:
         if ok:
             enabled_capabilities.append(capability)
+        # `dict` reasons for K8s and SSH will be printed in detail in
+        # _format_enabled_cloud. Skip here.
+        if not isinstance(reason, str):
+            continue
+        if ok:
             if reason is not None:
                 hints_to_capabilities.setdefault(reason, []).append(capability)
         elif reason is not None:
             reasons_to_capabilities.setdefault(reason, []).append(capability)
+    style_str = f'{colorama.Style.DIM}'
     status_msg: str = 'disabled'
-    styles: Dict[str, Any] = {'dim': True}
     capability_string: str = ''
+    detail_string: str = ''
     activated_account: Optional[str] = None
     if enabled_capabilities:
+        style_str = f'{colorama.Fore.GREEN}{colorama.Style.NORMAL}'
         status_msg = 'enabled'
-        styles = {'fg': 'green', 'bold': False}
         capability_string = f'[{", ".join(enabled_capabilities)}]'
         if verbose and cloud is not cloudflare:
             activated_account = cloud.get_active_user_identity_str()
-
+        if isinstance(cloud_tuple[1], (sky_clouds.SSH, sky_clouds.Kubernetes)):
+            detail_string = _format_context_details(cloud_tuple[1],
+                                                    show_details=True,
+                                                    ctx2text=ctx2text)
     echo(
-        click.style(f'  {cloud_repr}: {status_msg} {capability_string}',
-                    **styles))
+        click.style(
+            f'{style_str}  {cloud_repr}: {status_msg} {capability_string}'
+            f'{colorama.Style.RESET_ALL}{detail_string}'))
     if activated_account is not None:
         echo(f'    Activated account: {activated_account}')
     for reason, caps in hints_to_capabilities.items():
@@ -343,8 +363,77 @@ def _print_checked_cloud(
         echo(f'    Reason [{", ".join(caps)}]: {reason}')
 
 
+def _green_color(str_to_format: str) -> str:
+    return f'{colorama.Fore.GREEN}{str_to_format}{colorama.Style.RESET_ALL}'
+
+
+def _format_context_details(cloud: Union[str, sky_clouds.Cloud],
+                            show_details: bool,
+                            ctx2text: Optional[Dict[str, str]] = None) -> str:
+    if isinstance(cloud, str):
+        cloud_type = registry.CLOUD_REGISTRY.from_str(cloud)
+        assert cloud_type is not None
+    else:
+        cloud_type = cloud
+    if isinstance(cloud_type, sky_clouds.SSH):
+        # Get the cluster names by reading from the node pools file
+        contexts = sky_clouds.SSH.get_ssh_node_pool_contexts()
+    else:
+        assert isinstance(cloud_type, sky_clouds.Kubernetes)
+        contexts = sky_clouds.Kubernetes.existing_allowed_contexts()
+
+    filtered_contexts = []
+    for context in contexts:
+        if not show_details:
+            # Skip
+            if (ctx2text is None or context not in ctx2text or
+                    'disabled' in ctx2text[context]):
+                continue
+        filtered_contexts.append(context)
+
+    if not filtered_contexts:
+        return ''
+
+    def _red_color(str_to_format: str) -> str:
+        return (f'{colorama.Fore.LIGHTRED_EX}'
+                f'{str_to_format}'
+                f'{colorama.Style.RESET_ALL}')
+
+    def _dim_color(str_to_format: str) -> str:
+        return (f'{colorama.Style.DIM}'
+                f'{str_to_format}'
+                f'{colorama.Style.RESET_ALL}')
+
+    # Format the context info with consistent styling
+    contexts_formatted = []
+    for i, context in enumerate(filtered_contexts):
+        if isinstance(cloud_type, sky_clouds.SSH):
+            # TODO: This is a hack to remove the 'ssh-' prefix from the
+            # context name. Once we have a separate kubeconfig for SSH,
+            # this will not be required.
+            cleaned_context = context.lstrip('ssh-')
+        else:
+            cleaned_context = context
+        symbol = (ux_utils.INDENT_LAST_SYMBOL if i == len(filtered_contexts) -
+                  1 else ux_utils.INDENT_SYMBOL)
+        text_suffix = ''
+        if show_details:
+            if ctx2text is not None:
+                text_suffix = (
+                    f': {ctx2text[context]}' if context in ctx2text else
+                    (': ' + _red_color('disabled. ') +
+                     _dim_color('Reason: Not set up. Use `sky ssh up --infra '
+                                f'{context.lstrip("ssh-")}` to set up.')))
+        contexts_formatted.append(
+            f'\n    {symbol}{cleaned_context}{text_suffix}')
+    identity_str = ('SSH Node Pools' if isinstance(cloud_type, sky_clouds.SSH)
+                    else 'Allowed contexts')
+    return f'\n    {identity_str}:{"".join(contexts_formatted)}'
+
+
 def _format_enabled_cloud(cloud_name: str,
-                          capabilities: List[sky_cloud.CloudCapability]) -> str:
+                          capabilities: List[sky_cloud.CloudCapability],
+                          ctx2text: Optional[Dict[str, str]] = None) -> str:
     """Format the summary of enabled cloud and its enabled capabilities.
 
     Args:
@@ -355,34 +444,10 @@ def _format_enabled_cloud(cloud_name: str,
         A string of the formatted cloud and capabilities.
     """
     cloud_and_capabilities = f'{cloud_name} [{", ".join(capabilities)}]'
+    title = _green_color(cloud_and_capabilities)
 
-    def _green_color(str_to_format: str) -> str:
-        return (
-            f'{colorama.Fore.GREEN}{str_to_format}{colorama.Style.RESET_ALL}')
-
-    if cloud_name == repr(sky_clouds.Kubernetes()):
-        # Get enabled contexts for Kubernetes
-        existing_contexts = sky_clouds.Kubernetes.existing_allowed_contexts()
-        if not existing_contexts:
-            return _green_color(cloud_and_capabilities)
-
-        # Check if allowed_contexts is explicitly set in config
-        allowed_contexts = skypilot_config.get_nested(
-            ('kubernetes', 'allowed_contexts'), None)
-
-        # Format the context info with consistent styling
-        if allowed_contexts is not None:
-            contexts_formatted = []
-            for i, context in enumerate(existing_contexts):
-                symbol = (ux_utils.INDENT_LAST_SYMBOL
-                          if i == len(existing_contexts) -
-                          1 else ux_utils.INDENT_SYMBOL)
-                contexts_formatted.append(f'\n    {symbol}{context}')
-            context_info = f'  Allowed contexts:{"".join(contexts_formatted)}'
-        else:
-            context_info = f'  Active context: {existing_contexts[0]}'
+    if cloud_name in [repr(sky_clouds.Kubernetes()), repr(sky_clouds.SSH())]:
+        return (f'{title}' + _format_context_details(
+            cloud_name, show_details=False, ctx2text=ctx2text))
 
-        return (f'{_green_color(cloud_and_capabilities)}\n'
-                f'  {colorama.Style.DIM}{context_info}'
-                f'{colorama.Style.RESET_ALL}')
-    return _green_color(cloud_and_capabilities)
+    return title