skypilot-nightly 1.0.0.dev20250522__py3-none-any.whl → 1.0.0.dev20250524__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '417e7f813d86c03c334e9e12f38718766fc4daa5'
+_SKYPILOT_COMMIT_SHA = 'a5958f14156552e0347f0a7490b028473e7fd593'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250522'
+__version__ = '1.0.0.dev20250524'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

sky/adaptors/kubernetes.py

@@ -3,8 +3,8 @@ import logging
 import os
 from typing import Any, Callable, Optional, Set
 
+from sky import sky_logging
 from sky.adaptors import common
-from sky.sky_logging import set_logging_level
 from sky.utils import annotations
 from sky.utils import common_utils
 from sky.utils import ux_utils
@@ -26,6 +26,8 @@ DEFAULT_IN_CLUSTER_REGION = 'in-cluster'
 # set to DEFAULT_IN_CLUSTER_REGION.
 IN_CLUSTER_CONTEXT_NAME_ENV_VAR = 'SKYPILOT_IN_CLUSTER_CONTEXT_NAME'
 
+logger = sky_logging.init_logger(__name__)
+
 
 def _decorate_methods(obj: Any, decorator: Callable, decoration_type: str):
     for attr_name in dir(obj):
@@ -43,7 +45,7 @@ def _decorate_methods(obj: Any, decorator: Callable, decoration_type: str):
     return obj
 
 
-def _api_logging_decorator(logger: str, level: int):
+def _api_logging_decorator(logger_src: str, level: int):
     """Decorator to set logging level for API calls.
 
     This is used to suppress the verbose logging from urllib3 when calls to the
@@ -54,7 +56,9 @@ def _api_logging_decorator(logger: str, level: int):
 
         def wrapped(*args, **kwargs):
             obj = api(*args, **kwargs)
-            _decorate_methods(obj, set_logging_level(logger, level), 'api_log')
+            _decorate_methods(obj,
+                              sky_logging.set_logging_level(logger_src, level),
+                              'api_log')
             return obj
 
         return wrapped
@@ -71,27 +75,53 @@ def _load_config(context: Optional[str] = None):
         except kubernetes.config.config_exception.ConfigException as e:
             suffix = common_utils.format_exception(e, use_bracket=True)
             context_name = '(current-context)' if context is None else context
+            is_ssh_node_pool = False
+            if context_name.startswith('ssh-'):
+                context_name = context_name.lstrip('ssh-')
+                is_ssh_node_pool = True
             # Check if exception was due to no current-context
             if 'Expected key current-context' in str(e):
-                err_str = ('Failed to load Kubernetes configuration for '
-                           f'{context_name!r}. '
-                           'Kubeconfig does not contain any valid context(s).'
-                           f'\n{suffix}\n'
-                           '    If you were running a local Kubernetes '
-                           'cluster, run `sky local up` to start the cluster.')
+                if is_ssh_node_pool:
+                    context_name = context_name.lstrip('ssh-')
+                    err_str = ('Failed to load SSH Node Pool configuration for '
+                               f'{context_name!r}.\n'
+                               '    Run `sky ssh up --infra {context_name}` to '
+                               'set up or repair the cluster.')
+                else:
+                    err_str = (
+                        'Failed to load Kubernetes configuration for '
+                        f'{context_name!r}. '
+                        'Kubeconfig does not contain any valid context(s).'
+                        f'\n{suffix}\n'
+                        '    If you were running a local Kubernetes '
+                        'cluster, run `sky local up` to start the cluster.')
             else:
                 kubeconfig_path = os.environ.get('KUBECONFIG', '~/.kube/config')
-                err_str = (
-                    f'Failed to load Kubernetes configuration for '
-                    f'{context_name!r}. Please check if your kubeconfig file '
-                    f'exists at {kubeconfig_path} and is valid.\n{suffix}')
-            err_str += '\nTo disable Kubernetes for SkyPilot: run `sky check`.'
-            if context is None:  # kubernetes defaults to current-context.
+                if is_ssh_node_pool:
+                    err_str = (
+                        f'Failed to load SSH Node Pool configuration for '
+                        f'{context_name!r}. Run `sky ssh up --infra '
+                        f'{context_name}` to set up or repair the cluster.')
+                else:
+                    err_str = (
+                        'Failed to load Kubernetes configuration for '
+                        f'{context_name!r}. Please check if your kubeconfig '
+                        f'file exists at {kubeconfig_path} and is valid.'
+                        f'\n{suffix}\n')
+            if is_ssh_node_pool:
+                err_str += (f'\nTo disable SSH Node Pool {context_name!r}: '
+                            'run `sky check`.')
+            else:
                 err_str += (
                     '\nHint: Kubernetes attempted to query the current-context '
                     'set in kubeconfig. Check if the current-context is valid.')
             with ux_utils.print_exception_no_traceback():
-                raise ValueError(err_str) from None
+                if is_ssh_node_pool:
+                    # For SSH Node Pool, we don't want to surface k8s errors
+                    # (e.g., missing context) unless debug flag is set.
+                    logging.debug(f'Kubernetes error: {suffix}')
+                else:
+                    raise ValueError(err_str) from None
 
     if context == in_cluster_context_name() or context is None:
         try:

sky/backends/backend_utils.py

@@ -1556,6 +1556,16 @@ def check_owner_identity(cluster_name: str) -> None:
     handle = record['handle']
     if not isinstance(handle, backends.CloudVmRayResourceHandle):
         return
+    active_workspace = skypilot_config.get_active_workspace()
+    cluster_workspace = record.get('workspace',
+                                   constants.SKYPILOT_DEFAULT_WORKSPACE)
+    if active_workspace != cluster_workspace:
+        with ux_utils.print_exception_no_traceback():
+            raise exceptions.ClusterOwnerIdentityMismatchError(
+                f'{colorama.Fore.YELLOW}'
+                f'The cluster {cluster_name!r} is in workspace '
+                f'{cluster_workspace!r}, but the active workspace is '
+                f'{active_workspace!r}.{colorama.Fore.RESET}')
 
     launched_resources = handle.launched_resources.assert_launchable()
     cloud = launched_resources.cloud
@@ -2152,57 +2162,64 @@ def refresh_cluster_record(
     record = global_user_state.get_cluster_from_name(cluster_name)
     if record is None:
         return None
-    check_owner_identity(cluster_name)
-
-    if not isinstance(record['handle'], backends.CloudVmRayResourceHandle):
-        return record
-
-    # The loop logic allows us to notice if the status was updated in the
-    # global_user_state by another process and stop trying to get the lock.
-    # The core loop logic is adapted from FileLock's implementation.
-    lock = filelock.FileLock(CLUSTER_STATUS_LOCK_PATH.format(cluster_name))
-    start_time = time.perf_counter()
+    # TODO(zhwu, 05/20): switch to the specific workspace to make sure we are
+    # using the correct cloud credentials.
+    workspace = record.get('workspace', constants.SKYPILOT_DEFAULT_WORKSPACE)
+    with skypilot_config.local_active_workspace_ctx(workspace):
+        check_owner_identity(cluster_name)
 
-    # Loop until we have an up-to-date status or until we acquire the lock.
-    while True:
-        # Check to see if we can return the cached status.
-        if not _must_refresh_cluster_status(record, force_refresh_statuses):
+        if not isinstance(record['handle'], backends.CloudVmRayResourceHandle):
             return record
 
-        if not acquire_per_cluster_status_lock:
-            return _update_cluster_status(cluster_name)
+        # The loop logic allows us to notice if the status was updated in the
+        # global_user_state by another process and stop trying to get the lock.
+        # The core loop logic is adapted from FileLock's implementation.
+        lock = filelock.FileLock(CLUSTER_STATUS_LOCK_PATH.format(cluster_name))
+        start_time = time.perf_counter()
 
-        # Try to acquire the lock so we can fetch the status.
-        try:
-            with lock.acquire(blocking=False):
-                # Check the cluster status again, since it could have been
-                # updated between our last check and acquiring the lock.
-                record = global_user_state.get_cluster_from_name(cluster_name)
-                if record is None or not _must_refresh_cluster_status(
-                        record, force_refresh_statuses):
-                    return record
-                # Update and return the cluster status.
+        # Loop until we have an up-to-date status or until we acquire the lock.
+        while True:
+            # Check to see if we can return the cached status.
+            if not _must_refresh_cluster_status(record, force_refresh_statuses):
+                return record
+
+            if not acquire_per_cluster_status_lock:
                 return _update_cluster_status(cluster_name)
-        except filelock.Timeout:
-            # lock.acquire() will throw a Timeout exception if the lock is not
-            # available and we have blocking=False.
-            pass
-
-        # Logic adapted from FileLock.acquire().
-        # If cluster_status_lock_time is <0, we will never hit this. No timeout.
-        # Otherwise, if we have timed out, return the cached status. This has
-        # the potential to cause correctness issues, but if so it is the
-        # caller's responsibility to set the timeout to -1.
-        if 0 <= cluster_status_lock_timeout < time.perf_counter() - start_time:
-            logger.debug('Refreshing status: Failed get the lock for cluster '
-                         f'{cluster_name!r}. Using the cached status.')
-            return record
-        time.sleep(0.05)
 
-        # Refresh for next loop iteration.
-        record = global_user_state.get_cluster_from_name(cluster_name)
-        if record is None:
-            return None
+            # Try to acquire the lock so we can fetch the status.
+            try:
+                with lock.acquire(blocking=False):
+                    # Check the cluster status again, since it could have been
+                    # updated between our last check and acquiring the lock.
+                    record = global_user_state.get_cluster_from_name(
+                        cluster_name)
+                    if record is None or not _must_refresh_cluster_status(
+                            record, force_refresh_statuses):
+                        return record
+                    # Update and return the cluster status.
+                    return _update_cluster_status(cluster_name)
+            except filelock.Timeout:
+                # lock.acquire() will throw a Timeout exception if the lock is not
+                # available and we have blocking=False.
+                pass
+
+            # Logic adapted from FileLock.acquire().
+            # If cluster_status_lock_time is <0, we will never hit this. No timeout.
+            # Otherwise, if we have timed out, return the cached status. This has
+            # the potential to cause correctness issues, but if so it is the
+            # caller's responsibility to set the timeout to -1.
+            if 0 <= cluster_status_lock_timeout < time.perf_counter(
+            ) - start_time:
+                logger.debug(
+                    'Refreshing status: Failed get the lock for cluster '
+                    f'{cluster_name!r}. Using the cached status.')
+                return record
+            time.sleep(0.05)
+
+            # Refresh for next loop iteration.
+            record = global_user_state.get_cluster_from_name(cluster_name)
+            if record is None:
+                return None
 
 
 @timeline.event

sky/backends/cloud_vm_ray_backend.py

@@ -192,6 +192,7 @@ def _get_cluster_config_template(cloud):
         clouds.DO: 'do-ray.yml.j2',
         clouds.RunPod: 'runpod-ray.yml.j2',
         clouds.Kubernetes: 'kubernetes-ray.yml.j2',
+        clouds.SSH: 'kubernetes-ray.yml.j2',
         clouds.Vsphere: 'vsphere-ray.yml.j2',
         clouds.Vast: 'vast-ray.yml.j2',
         clouds.Fluidstack: 'fluidstack-ray.yml.j2',
@@ -1547,11 +1548,13 @@ class RetryingVmProvisioner(object):
                     controller_str = ('' if controller is None else
                                       f' {controller.value.name}')
                     if isinstance(to_provision.cloud, clouds.Kubernetes):
-                        # Omit the region name for Kubernetes.
+                        suffix = '.'
+                        if region.name.startswith('ssh-'):
+                            suffix = f' ({region.name.lstrip("ssh-")})'
                         logger.info(
                             ux_utils.starting_message(
                                 f'Launching{controller_str} on '
-                                f'{to_provision.cloud}.'))
+                                f'{to_provision.cloud}{suffix}'))
                     else:
                         logger.info(
                             ux_utils.starting_message(
@@ -1724,8 +1727,17 @@ class RetryingVmProvisioner(object):
                 f'{requested_resources}. ')
         elif to_provision.region is not None:
             # For public clouds, provision.region is always set.
-            message = ('Failed to acquire resources in all zones in '
-                       f'{to_provision.region} for {requested_resources}. ')
+            if to_provision.cloud.is_same_cloud(clouds.SSH()):
+                message = ('Failed to acquire resources in SSH Node Pool '
+                           f'({to_provision.region.lstrip("ssh-")}) for '
+                           f'{requested_resources}. The SSH Node Pool may not '
+                           'have enough resources.')
+            elif to_provision.cloud.is_same_cloud(clouds.Kubernetes()):
+                message = ('Failed to acquire resources in context '
+                           f'{to_provision.region} for {requested_resources}. ')
+            else:
+                message = ('Failed to acquire resources in all zones in '
+                           f'{to_provision.region} for {requested_resources}. ')
         else:
             message = (f'Failed to acquire resources in {to_provision.cloud} '
                        f'for {requested_resources}. ')
@@ -3495,7 +3507,9 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
                 # Add the managed job to job queue database.
                 managed_job_codegen = managed_jobs.ManagedJobCodeGen()
                 managed_job_code = managed_job_codegen.set_pending(
-                    job_id, managed_job_dag)
+                    job_id, managed_job_dag,
+                    skypilot_config.get_active_workspace(
+                        force_user_workspace=True))
                 # Set the managed job to PENDING state to make sure that this
                 # managed job appears in the `sky jobs queue`, even if it needs
                 # to wait to be submitted.