skypilot-nightly 1.0.0.dev20240925__py3-none-any.whl → 1.0.0.dev20240927__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = 'e13c39104cc3fca974e2afa207bcca24817f4e17'
+_SKYPILOT_COMMIT_SHA = 'e6b8d2c086544ab5cfdb877ad414eafddaa49cb4'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20240925'
+__version__ = '1.0.0.dev20240927'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

sky/adaptors/kubernetes.py

@@ -75,15 +75,17 @@ def _load_config(context: Optional[str] = None):
                 suffix += f' Error: {str(e)}'
             # Check if exception was due to no current-context
             if 'Expected key current-context' in str(e):
-                err_str = ('Failed to load Kubernetes configuration. '
-                           'Kubeconfig does not contain any valid context(s).'
-                           f'{suffix}\n'
-                           '    If you were running a local Kubernetes '
-                           'cluster, run `sky local up` to start the cluster.')
+                err_str = (
+                    f'Failed to load Kubernetes configuration for {context!r}. '
+                    'Kubeconfig does not contain any valid context(s).'
+                    f'{suffix}\n'
+                    '    If you were running a local Kubernetes '
+                    'cluster, run `sky local up` to start the cluster.')
             else:
-                err_str = ('Failed to load Kubernetes configuration. '
-                           'Please check if your kubeconfig file exists at '
-                           f'~/.kube/config and is valid.{suffix}')
+                err_str = (
+                    f'Failed to load Kubernetes configuration for {context!r}. '
+                    'Please check if your kubeconfig file exists at '
+                    f'~/.kube/config and is valid.{suffix}')
             err_str += '\nTo disable Kubernetes for SkyPilot: run `sky check`.'
             with ux_utils.print_exception_no_traceback():
                 raise ValueError(err_str) from None

sky/authentication.py

@@ -378,11 +378,11 @@ def setup_kubernetes_authentication(config: Dict[str, Any]) -> Dict[str, Any]:
     public_key_path = os.path.expanduser(PUBLIC_SSH_KEY_PATH)
     secret_name = clouds.Kubernetes.SKY_SSH_KEY_SECRET_NAME
     secret_field_name = clouds.Kubernetes().ssh_key_secret_field_name
-    namespace = config['provider'].get(
-        'namespace',
-        kubernetes_utils.get_current_kube_config_context_namespace())
     context = config['provider'].get(
         'context', kubernetes_utils.get_current_kube_config_context_name())
+    namespace = config['provider'].get(
+        'namespace',
+        kubernetes_utils.get_kube_config_context_namespace(context))
     k8s = kubernetes.kubernetes
     with open(public_key_path, 'r', encoding='utf-8') as f:
         public_key = f.read()
@@ -425,8 +425,8 @@ def setup_kubernetes_authentication(config: Dict[str, Any]) -> Dict[str, Any]:
             ssh_jump_name,
             nodeport_mode,
             private_key_path=private_key_path,
-            namespace=namespace,
-            context=context)
+            context=context,
+            namespace=namespace)
     elif network_mode == port_forward_mode:
         # Using `kubectl port-forward` creates a direct tunnel to the pod and
         # does not require a ssh jump pod.
@@ -441,7 +441,11 @@ def setup_kubernetes_authentication(config: Dict[str, Any]) -> Dict[str, Any]:
         #   on GKE.
         ssh_target = config['cluster_name'] + '-head'
         ssh_proxy_cmd = kubernetes_utils.get_ssh_proxy_command(
-            ssh_target, port_forward_mode, private_key_path=private_key_path)
+            ssh_target,
+            port_forward_mode,
+            private_key_path=private_key_path,
+            context=context,
+            namespace=namespace)
     else:
         # This should never happen because we check for this in from_str above.
         raise ValueError(f'Unsupported networking mode: {network_mode_str}')

sky/backends/backend_utils.py

@@ -428,6 +428,7 @@ class SSHConfigHelper(object):
               HostName {ip}
               User {username}
               IdentityFile {ssh_key_path}
+              AddKeysToAgent yes
               IdentitiesOnly yes
               ForwardAgent yes
               StrictHostKeyChecking no

sky/backends/cloud_vm_ray_backend.py

@@ -48,6 +48,7 @@ from sky.provision import common as provision_common
 from sky.provision import instance_setup
 from sky.provision import metadata_utils
 from sky.provision import provisioner
+from sky.provision.kubernetes import utils as kubernetes_utils
 from sky.skylet import autostop_lib
 from sky.skylet import constants
 from sky.skylet import job_lib
@@ -2081,7 +2082,7 @@ class CloudVmRayResourceHandle(backends.backend.ResourceHandle):
     """
     # Bump if any fields get added/removed/changed, and add backward
     # compaitibility logic in __setstate__.
-    _VERSION = 8
+    _VERSION = 9
 
     def __init__(
             self,
@@ -2515,6 +2516,19 @@ class CloudVmRayResourceHandle(backends.backend.ResourceHandle):
         if version < 8:
             self.cached_cluster_info = None
 
+        if version < 9:
+            # For backward compatibility, we should update the region of a
+            # SkyPilot cluster on Kubernetes to the actual context it is using.
+            # pylint: disable=import-outside-toplevel
+            launched_resources = state['launched_resources']
+            if isinstance(launched_resources.cloud, clouds.Kubernetes):
+                yaml_config = common_utils.read_yaml(
+                    os.path.expanduser(state['_cluster_yaml']))
+                context = kubernetes_utils.get_context_from_config(
+                    yaml_config['provider'])
+                state['launched_resources'] = launched_resources.copy(
+                    region=context)
+
         self.__dict__.update(state)
 
         # Because the update_cluster_ips and update_ssh_ports
@@ -4180,6 +4194,13 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
             global_user_state.set_cluster_autostop_value(
                 handle.cluster_name, idle_minutes_to_autostop, down)
 
+        # Add/Remove autodown annotations to/from Kubernetes pods.
+        if isinstance(handle.launched_resources.cloud, clouds.Kubernetes):
+            kubernetes_utils.set_autodown_annotations(
+                handle=handle,
+                idle_minutes_to_autostop=idle_minutes_to_autostop,
+                down=down)
+
     def is_definitely_autostopping(self,
                                    handle: CloudVmRayResourceHandle,
                                    stream_logs: bool = True) -> bool:

sky/cli.py

@@ -3026,14 +3026,11 @@ def show_gpus(
     kubernetes_is_enabled = sky_clouds.cloud_in_iterable(
         sky_clouds.Kubernetes(), global_user_state.get_cached_enabled_clouds())
 
-    if cloud_is_kubernetes and region is not None:
-        raise click.UsageError(
-            'The --region flag cannot be set with --cloud kubernetes.')
-
     def _list_to_str(lst):
         return ', '.join([str(e) for e in lst])
 
     def _get_kubernetes_realtime_gpu_table(
+            context: Optional[str] = None,
             name_filter: Optional[str] = None,
             quantity_filter: Optional[int] = None):
         if quantity_filter:
@@ -3048,7 +3045,7 @@ def show_gpus(
             gpus_only=True,
             clouds='kubernetes',
             name_filter=name_filter,
-            region_filter=region,
+            region_filter=context,
             quantity_filter=quantity_filter,
             case_sensitive=False)
         assert (set(counts.keys()) == set(capacity.keys()) == set(
@@ -3078,11 +3075,11 @@ def show_gpus(
             ])
         return realtime_gpu_table
 
-    def _get_kubernetes_node_info_table():
+    def _get_kubernetes_node_info_table(context: Optional[str]):
         node_table = log_utils.create_table(
             ['NODE_NAME', 'GPU_NAME', 'TOTAL_GPUS', 'FREE_GPUS'])
 
-        node_info_dict = kubernetes_utils.get_kubernetes_node_info()
+        node_info_dict = kubernetes_utils.get_kubernetes_node_info(context)
         for node_name, node_info in node_info_dict.items():
             node_table.add_row([
                 node_name, node_info.gpu_type,
@@ -3116,11 +3113,13 @@ def show_gpus(
             print_section_titles = False
             # If cloud is kubernetes, we want to show real-time capacity
             if kubernetes_is_enabled and (cloud is None or cloud_is_kubernetes):
+                context = region
                 try:
                     # If --cloud kubernetes is not specified, we want to catch
                     # the case where no GPUs are available on the cluster and
                     # print the warning at the end.
-                    k8s_realtime_table = _get_kubernetes_realtime_gpu_table()
+                    k8s_realtime_table = _get_kubernetes_realtime_gpu_table(
+                        context)
                 except ValueError as e:
                     if not cloud_is_kubernetes:
                         # Make it a note if cloud is not kubernetes
@@ -3129,9 +3128,10 @@ def show_gpus(
                 else:
                     print_section_titles = True
                     yield (f'{colorama.Fore.CYAN}{colorama.Style.BRIGHT}'
-                           f'Kubernetes GPUs{colorama.Style.RESET_ALL}\n')
+                           f'Kubernetes GPUs (Context: {context})'
+                           f'{colorama.Style.RESET_ALL}\n')
                     yield from k8s_realtime_table.get_string()
-                    k8s_node_table = _get_kubernetes_node_info_table()
+                    k8s_node_table = _get_kubernetes_node_info_table(context)
                     yield '\n\n'
                     yield (f'{colorama.Fore.CYAN}{colorama.Style.BRIGHT}'
                            f'Kubernetes per node GPU availability'

sky/clouds/kubernetes.py

@@ -1,4 +1,5 @@
 """Kubernetes."""
+import functools
 import json
 import os
 import re
@@ -52,8 +53,7 @@ class Kubernetes(clouds.Cloud):
     _DEFAULT_MEMORY_CPU_RATIO = 1
     _DEFAULT_MEMORY_CPU_RATIO_WITH_GPU = 4  # Allocate more memory for GPU tasks
     _REPR = 'Kubernetes'
-    _SINGLETON_REGION = 'kubernetes'
-    _regions: List[clouds.Region] = [clouds.Region(_SINGLETON_REGION)]
+    _LEGACY_SINGLETON_REGION = 'kubernetes'
     _CLOUD_UNSUPPORTED_FEATURES = {
         # TODO(romilb): Stopping might be possible to implement with
         #  container checkpointing introduced in Kubernetes v1.25. See:
@@ -88,8 +88,12 @@ class Kubernetes(clouds.Cloud):
         cls, resources: 'resources_lib.Resources'
     ) -> Dict[clouds.CloudImplementationFeatures, str]:
         unsupported_features = cls._CLOUD_UNSUPPORTED_FEATURES.copy()
+        context = resources.region
+        if context is None:
+            context = kubernetes_utils.get_current_kube_config_context_name()
         # Features to be disabled for exec auth
-        is_exec_auth, message = kubernetes_utils.is_kubeconfig_exec_auth()
+        is_exec_auth, message = kubernetes_utils.is_kubeconfig_exec_auth(
+            context)
         if is_exec_auth:
             assert isinstance(message, str), message
             # Controllers cannot spin up new pods with exec auth.
@@ -99,7 +103,7 @@ class Kubernetes(clouds.Cloud):
             unsupported_features[
                 clouds.CloudImplementationFeatures.AUTO_TERMINATE] = message
         # Allow spot instances if supported by the cluster
-        spot_label_key, _ = kubernetes_utils.get_spot_label()
+        spot_label_key, _ = kubernetes_utils.get_spot_label(context)
         if spot_label_key is not None:
             unsupported_features.pop(
                 clouds.CloudImplementationFeatures.SPOT_INSTANCE, None)
@@ -110,16 +114,87 @@ class Kubernetes(clouds.Cloud):
         return cls._MAX_CLUSTER_NAME_LEN_LIMIT
 
     @classmethod
-    def regions(cls) -> List[clouds.Region]:
-        return cls._regions
+    @functools.lru_cache(maxsize=1)
+    def _log_skipped_contexts_once(cls, skipped_contexts: Tuple[str,
+                                                                ...]) -> None:
+        """Log skipped contexts for only once.
+
+        We don't directly cache the result of _filter_existing_allowed_contexts
+        as the admin policy may update the allowed contexts.
+        """
+        if skipped_contexts:
+            logger.warning(
+                f'Kubernetes contexts {set(skipped_contexts)!r} specified in '
+                '"allowed_contexts" not found in kubeconfig. '
+                'Ignoring these contexts.')
+
+    @classmethod
+    def _existing_allowed_contexts(cls) -> List[str]:
+        """Get existing allowed contexts."""
+        all_contexts = kubernetes_utils.get_all_kube_config_context_names()
+        if all_contexts is None:
+            return []
+        all_contexts = set(all_contexts)
+
+        allowed_contexts = skypilot_config.get_nested(
+            ('kubernetes', 'allowed_contexts'), None)
+
+        if allowed_contexts is None:
+            current_context = (
+                kubernetes_utils.get_current_kube_config_context_name())
+            allowed_contexts = []
+            if current_context is not None:
+                allowed_contexts = [current_context]
+
+        existing_contexts = []
+        skipped_contexts = []
+        for context in allowed_contexts:
+            if context in all_contexts:
+                existing_contexts.append(context)
+            else:
+                skipped_contexts.append(context)
+        cls._log_skipped_contexts_once(tuple(skipped_contexts))
+        return existing_contexts
 
     @classmethod
     def regions_with_offering(cls, instance_type: Optional[str],
                               accelerators: Optional[Dict[str, int]],
                               use_spot: bool, region: Optional[str],
                               zone: Optional[str]) -> List[clouds.Region]:
-        # No notion of regions in Kubernetes - return a single region.
-        return cls.regions()
+        del accelerators, zone, use_spot  # unused
+        existing_contexts = cls._existing_allowed_contexts()
+
+        regions = [clouds.Region(context) for context in existing_contexts]
+
+        if region is not None:
+            regions = [r for r in regions if r.name == region]
+
+        # Check if requested instance type will fit in the cluster.
+        # TODO(zhwu,romilb): autoscaler type needs to be regional (per
+        # kubernetes cluster/context).
+        regions_to_return = []
+        autoscaler_type = kubernetes_utils.get_autoscaler_type()
+        if autoscaler_type is None and instance_type is not None:
+            # If autoscaler is not set, check if the instance type fits in the
+            # cluster. Else, rely on the autoscaler to provision the right
+            # instance type without running checks. Worst case, if autoscaling
+            # fails, the pod will be stuck in pending state until
+            # provision_timeout, after which failover will be triggered.
+            for r in regions:
+                context = r.name
+                fits, reason = kubernetes_utils.check_instance_fits(
+                    context, instance_type)
+                if fits:
+                    regions_to_return.append(r)
+                else:
+                    logger.debug(
+                        f'Instance type {instance_type} does '
+                        'not fit in the Kubernetes cluster with context: '
+                        f'{context}. Reason: {reason}')
+        else:
+            regions_to_return = regions
+
+        return regions_to_return
 
     def instance_type_to_hourly_cost(self,
                                      instance_type: str,
@@ -201,9 +276,9 @@ class Kubernetes(clouds.Cloud):
         accelerators: Optional[Dict[str, int]] = None,
         use_spot: bool = False,
     ) -> Iterator[Optional[List[clouds.Zone]]]:
-        del num_nodes, region, instance_type, accelerators, use_spot  # Unused.
-        for r in cls.regions():
-            yield r.zones
+        # Always yield None for zones, since Kubernetes does not have zones, and
+        # we should allow any region get to this point.
+        yield None
 
     @classmethod
     def get_zone_shell_cmd(cls) -> Optional[str]:
@@ -225,7 +300,10 @@ class Kubernetes(clouds.Cloud):
             dryrun: bool = False) -> Dict[str, Optional[str]]:
         del cluster_name, zones, dryrun  # Unused.
         if region is None:
-            region = self._regions[0]
+            context = kubernetes_utils.get_current_kube_config_context_name()
+        else:
+            context = region.name
+        assert context is not None, 'No context found in kubeconfig'
 
         r = resources
         acc_dict = self.get_accelerators_from_instance_type(r.instance_type)
@@ -244,9 +322,14 @@ class Kubernetes(clouds.Cloud):
         acc_count = k.accelerator_count if k.accelerator_count else 0
         acc_type = k.accelerator_type if k.accelerator_type else None
 
-        if resources.image_id is not None:
+        image_id_dict = resources.image_id
+        if image_id_dict is not None:
             # Use custom image specified in resources
-            image_id = resources.image_id['kubernetes']
+            if None in image_id_dict:
+                image_id = image_id_dict[None]
+            else:
+                assert resources.region in image_id_dict, image_id_dict
+                image_id = image_id_dict[resources.region]
             if image_id.startswith('docker:'):
                 image_id = image_id[len('docker:'):]
         else:
@@ -265,7 +348,7 @@ class Kubernetes(clouds.Cloud):
         # If GPUs are requested, set node label to match the GPU type.
         if acc_count > 0 and acc_type is not None:
             k8s_acc_label_key, k8s_acc_label_value = \
-                kubernetes_utils.get_gpu_label_key_value(acc_type)
+                kubernetes_utils.get_gpu_label_key_value(context, acc_type)
 
         port_mode = network_utils.get_port_mode(None)
 
@@ -309,13 +392,10 @@ class Kubernetes(clouds.Cloud):
         deploy_vars = {
             'instance_type': resources.instance_type,
             'custom_resources': custom_resources,
-            'region': region.name,
             'cpus': str(cpus),
             'memory': str(mem),
             'accelerator_count': str(acc_count),
             'timeout': str(timeout),
-            'k8s_namespace':
-                kubernetes_utils.get_current_kube_config_context_namespace(),
             'k8s_port_mode': port_mode.value,
             'k8s_networking_mode': network_utils.get_networking_mode().value,
             'k8s_ssh_key_secret_name': self.SKY_SSH_KEY_SECRET_NAME,
@@ -335,18 +415,30 @@ class Kubernetes(clouds.Cloud):
 
         # Add kubecontext if it is set. It may be None if SkyPilot is running
         # inside a pod with in-cluster auth.
-        curr_context = kubernetes_utils.get_current_kube_config_context_name()
-        if curr_context is not None:
-            deploy_vars['k8s_context'] = curr_context
+        if context is not None:
+            deploy_vars['k8s_context'] = context
+
+        namespace = kubernetes_utils.get_kube_config_context_namespace(context)
+        deploy_vars['k8s_namespace'] = namespace
 
         return deploy_vars
 
     def _get_feasible_launchable_resources(
         self, resources: 'resources_lib.Resources'
     ) -> 'resources_utils.FeasibleResources':
+        # TODO(zhwu): This needs to be updated to return the correct region
+        # (context) that has enough resources.
         fuzzy_candidate_list: List[str] = []
         if resources.instance_type is not None:
             assert resources.is_launchable(), resources
+            regions = self.regions_with_offering(
+                resources.instance_type,
+                accelerators=resources.accelerators,
+                use_spot=resources.use_spot,
+                region=resources.region,
+                zone=resources.zone)
+            if not regions:
+                return resources_utils.FeasibleResources([], [], None)
             resources = resources.copy(accelerators=None)
             return resources_utils.FeasibleResources([resources],
                                                      fuzzy_candidate_list, None)
@@ -391,34 +483,48 @@ class Kubernetes(clouds.Cloud):
                 kubernetes_utils.KubernetesInstanceType.from_resources(
                     gpu_task_cpus, gpu_task_memory, acc_count, acc_type).name)
 
-        # Check if requested instance type will fit in the cluster.
-        autoscaler_type = kubernetes_utils.get_autoscaler_type()
-        if autoscaler_type is None:
-            # If autoscaler is not set, check if the instance type fits in the
-            # cluster. Else, rely on the autoscaler to provision the right
-            # instance type without running checks. Worst case, if autoscaling
-            # fails, the pod will be stuck in pending state until
-            # provision_timeout, after which failover will be triggered.
-            fits, reason = kubernetes_utils.check_instance_fits(
-                chosen_instance_type)
-            if not fits:
-                logger.debug(f'Instance type {chosen_instance_type} does '
-                             'not fit in the Kubernetes cluster. '
-                             f'Reason: {reason}')
-                return resources_utils.FeasibleResources([], [], reason)
-
+        # Check the availability of the specified instance type in all contexts.
+        available_regions = self.regions_with_offering(
+            chosen_instance_type,
+            accelerators=None,
+            use_spot=resources.use_spot,
+            region=resources.region,
+            zone=resources.zone)
+        if not available_regions:
+            return resources_utils.FeasibleResources([], [], None)
         # No fuzzy lists for Kubernetes
+        # We don't set the resources returned with regions, because the
+        # optimizer will further find the valid region (context) for the
+        # resources.
         return resources_utils.FeasibleResources(_make([chosen_instance_type]),
                                                  [], None)
 
     @classmethod
     def check_credentials(cls) -> Tuple[bool, Optional[str]]:
         # Test using python API
-        try:
-            return kubernetes_utils.check_credentials()
-        except Exception as e:  # pylint: disable=broad-except
-            return (False, 'Credential check failed: '
-                    f'{common_utils.format_exception(e)}')
+        existing_allowed_contexts = cls._existing_allowed_contexts()
+        if not existing_allowed_contexts:
+            if skypilot_config.loaded_config_path() is None:
+                check_skypilot_config_msg = ''
+            else:
+                check_skypilot_config_msg = (
+                    ' and check "allowed_contexts" in your '
+                    f'{skypilot_config.loaded_config_path()} file.')
+            return (False, 'No available context found in kubeconfig. '
+                    'Check if you have a valid kubeconfig file' +
+                    check_skypilot_config_msg)
+        reasons = []
+        for context in existing_allowed_contexts:
+            try:
+                check_result = kubernetes_utils.check_credentials(context)
+                if check_result[0]:
+                    return check_result
+                reasons.append(f'{context}: {check_result[1]}')
+            except Exception as e:  # pylint: disable=broad-except
+                return (False, f'Credential check failed for {context}: '
+                        f'{common_utils.format_exception(e)}')
+        return (False, 'Failed to find available context with working '
+                'credentials. Details:\n' + '\n'.join(reasons))
 
     def get_credential_file_mounts(self) -> Dict[str, str]:
         if os.path.exists(os.path.expanduser(CREDENTIAL_PATH)):
@@ -433,10 +539,20 @@ class Kubernetes(clouds.Cloud):
             instance_type)
 
     def validate_region_zone(self, region: Optional[str], zone: Optional[str]):
-        if region != self._SINGLETON_REGION:
+        if region == self._LEGACY_SINGLETON_REGION:
+            # For backward compatibility, we allow the region to be set to the
+            # legacy singletonton region.
+            # TODO: Remove this after 0.9.0.
+            return region, zone
+
+        all_contexts = kubernetes_utils.get_all_kube_config_context_names()
+        if all_contexts is None:
+            all_contexts = []
+        if region not in all_contexts:
             raise ValueError(
-                'Kubernetes support does not support setting region.'
-                ' Cluster used is determined by the kubeconfig.')
+                f'Context {region} not found in kubeconfig. Kubernetes only '
+                'supports context names as regions. Available '
+                f'contexts: {all_contexts}')
         if zone is not None:
             raise ValueError('Kubernetes support does not support setting zone.'
                              ' Cluster used is determined by the kubeconfig.')

sky/clouds/oci.py

@@ -431,14 +431,17 @@ class OCI(clouds.Cloud):
 
     def get_credential_file_mounts(self) -> Dict[str, str]:
         """Returns a dict of credential file paths to mount paths."""
-        oci_cfg_file = oci_adaptor.get_config_file()
-        # Pass-in a profile parameter so that multiple profile in oci
-        # config file is supported (2023/06/09).
-        oci_cfg = oci_adaptor.get_oci_config(
-            profile=oci_utils.oci_config.get_profile())
-        api_key_file = oci_cfg[
-            'key_file'] if 'key_file' in oci_cfg else 'BadConf'
-        sky_cfg_file = oci_utils.oci_config.get_sky_user_config_file()
+        try:
+            oci_cfg_file = oci_adaptor.get_config_file()
+            # Pass-in a profile parameter so that multiple profile in oci
+            # config file is supported (2023/06/09).
+            oci_cfg = oci_adaptor.get_oci_config(
+                profile=oci_utils.oci_config.get_profile())
+            api_key_file = oci_cfg[
+                'key_file'] if 'key_file' in oci_cfg else 'BadConf'
+            sky_cfg_file = oci_utils.oci_config.get_sky_user_config_file()
+        except ImportError:
+            return {}
 
         # OCI config and API key file are mandatory
         credential_files = [oci_cfg_file, api_key_file]

sky/clouds/service_catalog/kubernetes_catalog.py

@@ -68,26 +68,35 @@ def list_accelerators_realtime(
     # TODO(romilb): This should be refactored to use get_kubernetes_node_info()
     #   function from kubernetes_utils.
     del all_regions, require_price  # Unused.
+    # TODO(zhwu): this should return all accelerators in multiple kubernetes
+    # clusters defined by allowed_contexts.
+    if region_filter is None:
+        context = kubernetes_utils.get_current_kube_config_context_name()
+    else:
+        context = region_filter
+    if context is None:
+        return {}, {}, {}
+
     k8s_cloud = Kubernetes()
     if not any(
             map(k8s_cloud.is_same_cloud,
                 sky_check.get_cached_enabled_clouds_or_refresh())
-    ) or not kubernetes_utils.check_credentials()[0]:
+    ) or not kubernetes_utils.check_credentials(context)[0]:
         return {}, {}, {}
 
-    has_gpu = kubernetes_utils.detect_gpu_resource()
+    has_gpu = kubernetes_utils.detect_gpu_resource(context)
     if not has_gpu:
         return {}, {}, {}
 
-    label_formatter, _ = kubernetes_utils.detect_gpu_label_formatter()
+    label_formatter, _ = kubernetes_utils.detect_gpu_label_formatter(context)
     if not label_formatter:
         return {}, {}, {}
 
     accelerators_qtys: Set[Tuple[str, int]] = set()
     key = label_formatter.get_label_key()
-    nodes = kubernetes_utils.get_kubernetes_nodes()
+    nodes = kubernetes_utils.get_kubernetes_nodes(context)
     # Get the pods to get the real-time GPU usage
-    pods = kubernetes_utils.get_kubernetes_pods()
+    pods = kubernetes_utils.get_all_pods_in_kubernetes_cluster(context)
     # Total number of GPUs in the cluster
     total_accelerators_capacity: Dict[str, int] = {}
     # Total number of GPUs currently available in the cluster
@@ -160,7 +169,7 @@ def list_accelerators_realtime(
                                     memory=None,
                                     price=0.0,
                                     spot_price=0.0,
-                                    region='kubernetes'))
+                                    region=context))
 
     df = pd.DataFrame(result,
                       columns=[
@@ -175,7 +184,6 @@ def list_accelerators_realtime(
     qtys_map = common.list_accelerators_impl('Kubernetes', df, gpus_only,
                                              name_filter, region_filter,
                                              quantity_filter, case_sensitive)
-
     return qtys_map, total_accelerators_capacity, total_accelerators_available