skypilot-nightly 1.0.0.dev20250627__py3-none-any.whl → 1.0.0.dev20250630__py3-none-any.whl

sky/provision/__init__.py

@@ -6,7 +6,7 @@ providers supported by SkyPilot need to follow.
 import functools
 import inspect
 import typing
-from typing import Any, Dict, List, Optional, Type
+from typing import Any, Dict, List, Optional, Tuple, Type
 
 from sky import models
 from sky import sky_logging
@@ -106,7 +106,7 @@ def bootstrap_instances(
 
 @_route_to_cloud_impl
 def apply_volume(provider_name: str,
-                 config: models.VolumeConfig) -> models.VolumeConfig:
+                 volume_config: models.VolumeConfig) -> models.VolumeConfig:
     """Create or register a volume.
 
     This function creates or registers a volume with the provided configuration,
@@ -117,15 +117,23 @@ def apply_volume(provider_name: str,
 
 @_route_to_cloud_impl
 def delete_volume(provider_name: str,
-                  config: models.VolumeConfig) -> models.VolumeConfig:
+                  volume_config: models.VolumeConfig) -> models.VolumeConfig:
     """Delete a volume."""
     raise NotImplementedError
 
 
 @_route_to_cloud_impl
-def get_volume_usedby(provider_name: str,
-                      config: models.VolumeConfig) -> List[str]:
-    """Get the usedby of a volume."""
+def get_volume_usedby(
+    provider_name: str,
+    volume_config: models.VolumeConfig,
+) -> Tuple[List[str], List[str]]:
+    """Get the usedby of a volume.
+
+    Returns:
+        usedby_pods: List of pods using the volume. These may include pods
+                     not created by SkyPilot.
+        usedby_clusters: List of clusters using the volume.
+    """
     raise NotImplementedError
 
 

sky/provision/kubernetes/config.py

@@ -35,7 +35,7 @@ def bootstrap_instances(
     _configure_services(namespace, context, config.provider_config)
 
     networking_mode = network_utils.get_networking_mode(
-        config.provider_config.get('networking_mode'))
+        config.provider_config.get('networking_mode'), context)
     if networking_mode == kubernetes_enums.KubernetesNetworkingMode.NODEPORT:
         config = _configure_ssh_jump(namespace, context, config)
 

sky/provision/kubernetes/constants.py

@@ -15,3 +15,12 @@ SKY_K8S_EXEC_AUTH_PATH = '$HOME/skypilot-runtime/bin:$HOME/google-cloud-sdk/bin:
 
 # cache directory for kubeconfig with modified exec auth
 SKY_K8S_EXEC_AUTH_KUBECONFIG_CACHE = '~/.sky/generated/kubeconfigs'
+
+# Labels for the Pods created by SkyPilot
+TAG_RAY_CLUSTER_NAME = 'ray-cluster-name'
+TAG_SKYPILOT_CLUSTER_NAME = 'skypilot-cluster-name'
+TAG_POD_INITIALIZED = 'skypilot-initialized'
+TAG_SKYPILOT_DEPLOYMENT_NAME = 'skypilot-deployment-name'
+
+# Pod phases that are not holding PVCs
+PVC_NOT_HOLD_POD_PHASES = ['Succeeded', 'Failed']

sky/provision/kubernetes/instance.py

@@ -12,6 +12,7 @@ from sky.provision import common
 from sky.provision import constants
 from sky.provision import docker_utils
 from sky.provision.kubernetes import config as config_lib
+from sky.provision.kubernetes import constants as k8s_constants
 from sky.provision.kubernetes import network_utils
 from sky.provision.kubernetes import utils as kubernetes_utils
 from sky.provision.kubernetes import volume
@@ -30,14 +31,10 @@ _MAX_RETRIES = 3
 _NUM_THREADS = subprocess_utils.get_parallel_threads('kubernetes')
 
 logger = sky_logging.init_logger(__name__)
-TAG_RAY_CLUSTER_NAME = 'ray-cluster-name'
-TAG_SKYPILOT_CLUSTER_NAME = 'skypilot-cluster-name'
-TAG_POD_INITIALIZED = 'skypilot-initialized'
-TAG_SKYPILOT_DEPLOYMENT_NAME = 'skypilot-deployment-name'
 
 
 def ray_tag_filter(cluster_name: str) -> Dict[str, str]:
-    return {TAG_RAY_CLUSTER_NAME: cluster_name}
+    return {k8s_constants.TAG_RAY_CLUSTER_NAME: cluster_name}
 
 
 def _is_head(pod) -> bool:
@@ -75,7 +72,8 @@ def is_high_availability_cluster_by_kubectl(
         deployment_list = kubernetes.apps_api(
             context).list_namespaced_deployment(
                 namespace,
-                label_selector=f'{TAG_SKYPILOT_CLUSTER_NAME}={cluster_name}')
+                label_selector=
+                f'{k8s_constants.TAG_SKYPILOT_CLUSTER_NAME}={cluster_name}')
     except kubernetes.api_exception():
         return False
     # It is a high availability cluster if there is at least one deployment
@@ -280,10 +278,12 @@ def _wait_for_pods_to_schedule(namespace, context, new_nodes, timeout: int):
     while _evaluate_timeout():
         # Get all pods in a single API call using the cluster name label
         # which all pods in new_nodes should share
-        cluster_name = new_nodes[0].metadata.labels[TAG_SKYPILOT_CLUSTER_NAME]
+        cluster_name = new_nodes[0].metadata.labels[
+            k8s_constants.TAG_SKYPILOT_CLUSTER_NAME]
         pods = kubernetes.core_api(context).list_namespaced_pod(
             namespace,
-            label_selector=f'{TAG_SKYPILOT_CLUSTER_NAME}={cluster_name}').items
+            label_selector=
+            f'{k8s_constants.TAG_SKYPILOT_CLUSTER_NAME}={cluster_name}').items
 
         # Get the set of found pod names and check if we have all expected pods
         found_pod_names = {pod.metadata.name for pod in pods}
@@ -361,10 +361,12 @@ def _wait_for_pods_to_run(namespace, context, new_nodes):
 
     while True:
         # Get all pods in a single API call
-        cluster_name = new_nodes[0].metadata.labels[TAG_SKYPILOT_CLUSTER_NAME]
+        cluster_name = new_nodes[0].metadata.labels[
+            k8s_constants.TAG_SKYPILOT_CLUSTER_NAME]
         all_pods = kubernetes.core_api(context).list_namespaced_pod(
             namespace,
-            label_selector=f'{TAG_SKYPILOT_CLUSTER_NAME}={cluster_name}').items
+            label_selector=
+            f'{k8s_constants.TAG_SKYPILOT_CLUSTER_NAME}={cluster_name}').items
 
         # Get the set of found pod names and check if we have all expected pods
         found_pod_names = {pod.metadata.name for pod in all_pods}
@@ -732,7 +734,7 @@ def _create_pods(region: str, cluster_name_on_cloud: str,
     else:
         pod_spec['metadata']['labels'] = tags
     pod_spec['metadata']['labels'].update(
-        {TAG_SKYPILOT_CLUSTER_NAME: cluster_name_on_cloud})
+        {k8s_constants.TAG_SKYPILOT_CLUSTER_NAME: cluster_name_on_cloud})
 
     terminating_pods = kubernetes_utils.filter_pods(namespace, context, tags,
                                                     ['Terminating'])
@@ -841,7 +843,7 @@ def _create_pods(region: str, cluster_name_on_cloud: str,
                 'podAffinityTerm': {
                     'labelSelector': {
                         'matchExpressions': [{
-                            'key': TAG_SKYPILOT_CLUSTER_NAME,
+                            'key': k8s_constants.TAG_SKYPILOT_CLUSTER_NAME,
                             'operator': 'In',
                             'values': [cluster_name_on_cloud]
                         }]
@@ -884,7 +886,7 @@ def _create_pods(region: str, cluster_name_on_cloud: str,
             # Add the deployment name as a label to the pod spec
             deployment_name = deployment_spec['metadata']['name']
             pod_spec_copy['metadata']['labels'][
-                TAG_SKYPILOT_DEPLOYMENT_NAME] = deployment_name
+                k8s_constants.TAG_SKYPILOT_DEPLOYMENT_NAME] = deployment_name
             template_pod_spec['metadata'] = pod_spec_copy['metadata']
             template_pod_spec['spec'].update(pod_spec_copy['spec'])
             # Propagate the labels to the deployment for identification.
@@ -941,7 +943,7 @@ def _create_pods(region: str, cluster_name_on_cloud: str,
             head_pod_name = pod.metadata.name
 
     networking_mode = network_utils.get_networking_mode(
-        config.provider_config.get('networking_mode'))
+        config.provider_config.get('networking_mode'), context)
     if networking_mode == kubernetes_enums.KubernetesNetworkingMode.NODEPORT:
         # Adding the jump pod to the new_nodes list as well so it can be
         # checked if it's scheduled and running along with other pods.
@@ -1102,7 +1104,7 @@ def terminate_instances(
 
     # Clean up the SSH jump pod if in use
     networking_mode = network_utils.get_networking_mode(
-        provider_config.get('networking_mode'))
+        provider_config.get('networking_mode'), context)
     if networking_mode == kubernetes_enums.KubernetesNetworkingMode.NODEPORT:
         pod_name = list(pods.keys())[0]
         try:
@@ -1147,8 +1149,11 @@ def get_cluster_info(
     head_pod_name = None
 
     port_forward_mode = kubernetes_enums.KubernetesNetworkingMode.PORTFORWARD
-    network_mode_str = skypilot_config.get_nested(('kubernetes', 'networking'),
-                                                  port_forward_mode.value)
+    network_mode_str = skypilot_config.get_effective_region_config(
+        cloud='kubernetes',
+        region=context,
+        keys=('networking_mode',),
+        default_value=port_forward_mode.value)
     network_mode = kubernetes_enums.KubernetesNetworkingMode.from_str(
         network_mode_str)
     external_ip = kubernetes_utils.get_external_ip(network_mode, context)
@@ -1286,7 +1291,8 @@ def get_command_runners(
 
         # Try to get deployment name from label first
         head_instance_info = instances[pod_name][0]
-        deployment = head_instance_info.tags.get(TAG_SKYPILOT_DEPLOYMENT_NAME)
+        deployment = head_instance_info.tags.get(
+            k8s_constants.TAG_SKYPILOT_DEPLOYMENT_NAME)
 
         node_list = [((namespace, context), pod_name)]
         head_runner = command_runner.KubernetesCommandRunner(

sky/provision/kubernetes/network.py

@@ -22,8 +22,9 @@ def open_ports(
 ) -> None:
     """See sky/provision/__init__.py"""
     assert provider_config is not None, 'provider_config is required'
+    context = kubernetes_utils.get_context_from_config(provider_config)
     port_mode = network_utils.get_port_mode(
-        provider_config.get('port_mode', None))
+        provider_config.get('port_mode', None), context)
     ports = list(port_ranges_to_set(ports))
     if port_mode == kubernetes_enums.KubernetesPortMode.LOADBALANCER:
         _open_ports_using_loadbalancer(
@@ -46,8 +47,10 @@ def _open_ports_using_loadbalancer(
 ) -> None:
     service_name = _LOADBALANCER_SERVICE_NAME.format(
         cluster_name_on_cloud=cluster_name_on_cloud)
+    context = kubernetes_utils.get_context_from_config(provider_config)
     content = network_utils.fill_loadbalancer_template(
         namespace=provider_config.get('namespace', 'default'),
+        context=context,
         service_name=service_name,
         ports=ports,
         selector_key='skypilot-cluster',
@@ -59,7 +62,7 @@ def _open_ports_using_loadbalancer(
 
     network_utils.create_or_replace_namespaced_service(
         namespace=kubernetes_utils.get_namespace_from_config(provider_config),
-        context=kubernetes_utils.get_context_from_config(provider_config),
+        context=context,
         service_name=service_name,
         service_spec=content['service_spec'])
 
@@ -70,6 +73,7 @@ def _open_ports_using_ingress(
     provider_config: Dict[str, Any],
 ) -> None:
     context = kubernetes_utils.get_context_from_config(provider_config)
+    namespace = kubernetes_utils.get_namespace_from_config(provider_config)
     # Check if an ingress controller exists
     if not network_utils.ingress_controller_exists(context):
         raise Exception(
@@ -100,6 +104,7 @@ def _open_ports_using_ingress(
     # multiple rules.
     content = network_utils.fill_ingress_template(
         namespace=provider_config.get('namespace', 'default'),
+        context=context,
         service_details=service_details,
         ingress_name=f'{cluster_name_on_cloud}-skypilot-ingress',
         selector_key='skypilot-cluster',
@@ -111,9 +116,8 @@ def _open_ports_using_ingress(
         # Update metadata from config
         kubernetes_utils.merge_custom_metadata(service_spec['metadata'])
         network_utils.create_or_replace_namespaced_service(
-            namespace=kubernetes_utils.get_namespace_from_config(
-                provider_config),
-            context=kubernetes_utils.get_context_from_config(provider_config),
+            namespace=namespace,
+            context=context,
             service_name=service_name,
             service_spec=service_spec,
         )
@@ -121,8 +125,8 @@ def _open_ports_using_ingress(
     kubernetes_utils.merge_custom_metadata(content['ingress_spec']['metadata'])
     # Create or update the single ingress for all services
     network_utils.create_or_replace_namespaced_ingress(
-        namespace=kubernetes_utils.get_namespace_from_config(provider_config),
-        context=kubernetes_utils.get_context_from_config(provider_config),
+        namespace=namespace,
+        context=context,
         ingress_name=f'{cluster_name_on_cloud}-skypilot-ingress',
         ingress_spec=content['ingress_spec'],
     )
@@ -135,8 +139,9 @@ def cleanup_ports(
 ) -> None:
     """See sky/provision/__init__.py"""
     assert provider_config is not None, 'provider_config is required'
+    context = kubernetes_utils.get_context_from_config(provider_config)
     port_mode = network_utils.get_port_mode(
-        provider_config.get('port_mode', None))
+        provider_config.get('port_mode', None), context)
     ports = list(port_ranges_to_set(ports))
     if port_mode == kubernetes_enums.KubernetesPortMode.LOADBALANCER:
         _cleanup_ports_for_loadbalancer(
@@ -202,8 +207,9 @@ def query_ports(
     """See sky/provision/__init__.py"""
     del head_ip  # unused
     assert provider_config is not None, 'provider_config is required'
+    context = kubernetes_utils.get_context_from_config(provider_config)
     port_mode = network_utils.get_port_mode(
-        provider_config.get('port_mode', None))
+        provider_config.get('port_mode', None), context)
     ports = list(port_ranges_to_set(ports))
 
     try:

sky/provision/kubernetes/network_utils.py

@@ -28,7 +28,8 @@ _LOADBALANCER_TEMPLATE_NAME = 'kubernetes-loadbalancer.yml.j2'
 
 
 def get_port_mode(
-        mode_str: Optional[str] = None) -> kubernetes_enums.KubernetesPortMode:
+        mode_str: Optional[str],
+        context: Optional[str]) -> kubernetes_enums.KubernetesPortMode:
     """Get the port mode from the provider config."""
 
     curr_kube_config = kubernetes_utils.get_current_kube_config_context_name()
@@ -38,9 +39,11 @@ def get_port_mode(
         # If running in kind (`sky local up`), use ingress mode
         return kubernetes_enums.KubernetesPortMode.INGRESS
 
-    mode_str = mode_str or skypilot_config.get_nested(
-        ('kubernetes', 'ports'),
-        kubernetes_enums.KubernetesPortMode.LOADBALANCER.value)
+    mode_str = mode_str or skypilot_config.get_effective_region_config(
+        cloud='kubernetes',
+        region=context,
+        keys=('ports',),
+        default_value=kubernetes_enums.KubernetesPortMode.LOADBALANCER.value)
     try:
         port_mode = kubernetes_enums.KubernetesPortMode(mode_str)
     except ValueError as e:
@@ -54,12 +57,16 @@ def get_port_mode(
 
 
 def get_networking_mode(
-    mode_str: Optional[str] = None
+    mode_str: Optional[str],
+    context: Optional[str],
 ) -> kubernetes_enums.KubernetesNetworkingMode:
     """Get the networking mode from the provider config."""
-    mode_str = mode_str or skypilot_config.get_nested(
-        ('kubernetes', 'networking_mode'),
-        kubernetes_enums.KubernetesNetworkingMode.PORTFORWARD.value)
+    mode_str = mode_str or skypilot_config.get_effective_region_config(
+        cloud='kubernetes',
+        region=context,
+        keys=('networking_mode',),
+        default_value=kubernetes_enums.KubernetesNetworkingMode.PORTFORWARD.
+        value)
     try:
         networking_mode = kubernetes_enums.KubernetesNetworkingMode.from_str(
             mode_str)
@@ -70,9 +77,9 @@ def get_networking_mode(
     return networking_mode
 
 
-def fill_loadbalancer_template(namespace: str, service_name: str,
-                               ports: List[int], selector_key: str,
-                               selector_value: str) -> Dict:
+def fill_loadbalancer_template(namespace: str, context: Optional[str],
+                               service_name: str, ports: List[int],
+                               selector_key: str, selector_value: str) -> Dict:
     template_path = os.path.join(sky.__root_dir__, 'templates',
                                  _LOADBALANCER_TEMPLATE_NAME)
     if not os.path.exists(template_path):
@@ -81,10 +88,16 @@ def fill_loadbalancer_template(namespace: str, service_name: str,
 
     with open(template_path, 'r', encoding='utf-8') as fin:
         template = fin.read()
-    annotations = skypilot_config.get_nested(
-        ('kubernetes', 'custom_metadata', 'annotations'), {})
-    labels = skypilot_config.get_nested(
-        ('kubernetes', 'custom_metadata', 'labels'), {})
+    annotations = skypilot_config.get_effective_region_config(
+        cloud='kubernetes',
+        region=context,
+        keys=('custom_metadata', 'annotations'),
+        default_value={})
+    labels = skypilot_config.get_effective_region_config(
+        cloud='kubernetes',
+        region=context,
+        keys=('custom_metadata', 'labels'),
+        default_value={})
     j2_template = jinja2.Template(template)
     cont = j2_template.render(
         namespace=namespace,
@@ -99,10 +112,10 @@ def fill_loadbalancer_template(namespace: str, service_name: str,
     return content
 
 
-def fill_ingress_template(namespace: str, service_details: List[Tuple[str, int,
-                                                                      str]],
-                          ingress_name: str, selector_key: str,
-                          selector_value: str) -> Dict:
+def fill_ingress_template(namespace: str, context: Optional[str],
+                          service_details: List[Tuple[str, int,
+                                                      str]], ingress_name: str,
+                          selector_key: str, selector_value: str) -> Dict:
     template_path = os.path.join(sky.__root_dir__, 'templates',
                                  _INGRESS_TEMPLATE_NAME)
     if not os.path.exists(template_path):
@@ -110,10 +123,16 @@ def fill_ingress_template(namespace: str, service_details: List[Tuple[str, int,
             f'Template "{_INGRESS_TEMPLATE_NAME}" does not exist.')
     with open(template_path, 'r', encoding='utf-8') as fin:
         template = fin.read()
-    annotations = skypilot_config.get_nested(
-        ('kubernetes', 'custom_metadata', 'annotations'), {})
-    labels = skypilot_config.get_nested(
-        ('kubernetes', 'custom_metadata', 'labels'), {})
+    annotations = skypilot_config.get_effective_region_config(
+        cloud='kubernetes',
+        region=context,
+        keys=('custom_metadata', 'annotations'),
+        default_value={})
+    labels = skypilot_config.get_effective_region_config(
+        cloud='kubernetes',
+        region=context,
+        keys=('custom_metadata', 'labels'),
+        default_value={})
     j2_template = jinja2.Template(template)
     cont = j2_template.render(
         namespace=namespace,

sky/provision/kubernetes/utils.py

@@ -1190,7 +1190,11 @@ def get_accelerator_label_key_values(
     context_display_name = common_utils.removeprefix(
         context, 'ssh-') if (context and is_ssh_node_pool) else context
 
-    autoscaler_type = get_autoscaler_type()
+    autoscaler_type = skypilot_config.get_effective_region_config(
+        cloud='kubernetes',
+        region=context,
+        keys=('autoscaler',),
+        default_value=None)
     if autoscaler_type is not None:
         # If autoscaler is set in config.yaml, override the label key and value
         # to the autoscaler's format and bypass the GPU checks.
@@ -1595,9 +1599,11 @@ def is_kubeconfig_exec_auth(
     user_details = next(
         user for user in user_details if user['name'] == target_username)
 
-    remote_identity = skypilot_config.get_nested(
-        ('kubernetes', 'remote_identity'),
-        schemas.get_default_remote_identity('kubernetes'))
+    remote_identity = skypilot_config.get_effective_region_config(
+        cloud='kubernetes',
+        region=context,
+        keys=('remote_identity',),
+        default_value=schemas.get_default_remote_identity('kubernetes'))
     if ('exec' in user_details.get('user', {}) and remote_identity
             == schemas.RemoteIdentityOptions.LOCAL_CREDENTIALS.value):
         ctx_name = context_obj['name']
@@ -2078,7 +2084,7 @@ def setup_ssh_jump_svc(ssh_jump_name: str, namespace: str,
     content = fill_ssh_jump_template('', '', ssh_jump_name, service_type.value)
 
     # Add custom metadata from config
-    merge_custom_metadata(content['service_spec']['metadata'])
+    merge_custom_metadata(content['service_spec']['metadata'], context)
 
     # Create service
     try:
@@ -2158,7 +2164,7 @@ def setup_ssh_jump_pod(ssh_jump_name: str, ssh_jump_image: str,
 
     # Add custom metadata to all objects
     for object_type in content.keys():
-        merge_custom_metadata(content[object_type]['metadata'])
+        merge_custom_metadata(content[object_type]['metadata'], context)
 
     # ServiceAccount
     try:
@@ -2370,7 +2376,7 @@ def check_port_forward_mode_dependencies(
     return None
 
 
-def get_endpoint_debug_message() -> str:
+def get_endpoint_debug_message(context: Optional[str] = None) -> str:
     """ Returns a string message for user to debug Kubernetes port opening
 
     Polls the configured ports mode on Kubernetes to produce an
@@ -2378,7 +2384,7 @@ def get_endpoint_debug_message() -> str:
 
     Also checks if the
     """
-    port_mode = network_utils.get_port_mode()
+    port_mode = network_utils.get_port_mode(None, context)
     if port_mode == kubernetes_enums.KubernetesPortMode.INGRESS:
         endpoint_type = 'Ingress'
         debug_cmd = 'kubectl describe ingress && kubectl describe ingressclass'
@@ -2396,6 +2402,7 @@ def combine_pod_config_fields(
     cluster_yaml_path: str,
     cluster_config_overrides: Dict[str, Any],
     cloud: Optional[clouds.Cloud] = None,
+    context: Optional[str] = None,
 ) -> None:
     """Adds or updates fields in the YAML with fields from the
     ~/.sky/config.yaml's kubernetes.pod_spec dict.
@@ -2438,19 +2445,28 @@ def combine_pod_config_fields(
     with open(cluster_yaml_path, 'r', encoding='utf-8') as f:
         yaml_content = f.read()
     yaml_obj = yaml.safe_load(yaml_content)
-    # We don't use override_configs in `skypilot_config.get_nested`, as merging
+    # We don't use override_configs in `get_effective_region_config`, as merging
     # the pod config requires special handling.
     if isinstance(cloud, clouds.SSH):
-        kubernetes_config = skypilot_config.get_nested(('ssh', 'pod_config'),
-                                                       default_value={},
-                                                       override_configs={})
-        override_pod_config = (cluster_config_overrides.get('ssh', {}).get(
-            'pod_config', {}))
+        kubernetes_config = skypilot_config.get_effective_region_config(
+            cloud='ssh', region=None, keys=('pod_config',), default_value={})
+        override_pod_config = config_utils.get_cloud_config_value_from_dict(
+            dict_config=cluster_config_overrides,
+            cloud='ssh',
+            keys=('pod_config',),
+            default_value={})
     else:
-        kubernetes_config = skypilot_config.get_nested(
-            ('kubernetes', 'pod_config'), default_value={}, override_configs={})
-        override_pod_config = (cluster_config_overrides.get(
-            'kubernetes', {}).get('pod_config', {}))
+        kubernetes_config = skypilot_config.get_effective_region_config(
+            cloud='kubernetes',
+            region=context,
+            keys=('pod_config',),
+            default_value={})
+        override_pod_config = config_utils.get_cloud_config_value_from_dict(
+            dict_config=cluster_config_overrides,
+            cloud='kubernetes',
+            region=context,
+            keys=('pod_config',),
+            default_value={})
     config_utils.merge_k8s_configs(kubernetes_config, override_pod_config)
 
     # Merge the kubernetes config into the YAML for both head and worker nodes.
@@ -2462,7 +2478,8 @@ def combine_pod_config_fields(
     common_utils.dump_yaml(cluster_yaml_path, yaml_obj)
 
 
-def combine_metadata_fields(cluster_yaml_path: str) -> None:
+def combine_metadata_fields(cluster_yaml_path: str,
+                            context: Optional[str] = None) -> None:
     """Updates the metadata for all Kubernetes objects created by SkyPilot with
     fields from the ~/.sky/config.yaml's kubernetes.custom_metadata dict.
 
@@ -2472,8 +2489,11 @@ def combine_metadata_fields(cluster_yaml_path: str) -> None:
     with open(cluster_yaml_path, 'r', encoding='utf-8') as f:
         yaml_content = f.read()
     yaml_obj = yaml.safe_load(yaml_content)
-    custom_metadata = skypilot_config.get_nested(
-        ('kubernetes', 'custom_metadata'), {})
+    custom_metadata = skypilot_config.get_effective_region_config(
+        cloud='kubernetes',
+        region=context,
+        keys=('custom_metadata',),
+        default_value={})
 
     # List of objects in the cluster YAML to be updated
     combination_destinations = [
@@ -2496,13 +2516,17 @@ def combine_metadata_fields(cluster_yaml_path: str) -> None:
     common_utils.dump_yaml(cluster_yaml_path, yaml_obj)
 
 
-def merge_custom_metadata(original_metadata: Dict[str, Any]) -> None:
+def merge_custom_metadata(original_metadata: Dict[str, Any],
+                          context: Optional[str] = None) -> None:
     """Merges original metadata with custom_metadata from config
 
     Merge is done in-place, so return is not required
     """
-    custom_metadata = skypilot_config.get_nested(
-        ('kubernetes', 'custom_metadata'), {})
+    custom_metadata = skypilot_config.get_effective_region_config(
+        cloud='kubernetes',
+        region=context,
+        keys=('custom_metadata',),
+        default_value={})
     config_utils.merge_k8s_configs(original_metadata, custom_metadata)
 
 
@@ -2556,7 +2580,7 @@ def create_namespace(namespace: str, context: Optional[str]) -> None:
         return
 
     ns_metadata = dict(name=namespace, labels={'parent': 'skypilot'})
-    merge_custom_metadata(ns_metadata)
+    merge_custom_metadata(ns_metadata, context)
     namespace_obj = kubernetes_client.V1Namespace(metadata=ns_metadata)
     try:
         kubernetes.core_api(context).create_namespace(namespace_obj)
@@ -2582,15 +2606,14 @@ def get_head_pod_name(cluster_name_on_cloud: str):
     return f'{cluster_name_on_cloud}-head'
 
 
-def get_autoscaler_type(
-) -> Optional[kubernetes_enums.KubernetesAutoscalerType]:
-    """Returns the autoscaler type by reading from config"""
-    autoscaler_type = skypilot_config.get_nested(('kubernetes', 'autoscaler'),
-                                                 None)
-    if autoscaler_type is not None:
-        autoscaler_type = kubernetes_enums.KubernetesAutoscalerType(
-            autoscaler_type)
-    return autoscaler_type
+def get_custom_config_k8s_contexts() -> List[str]:
+    """Returns the list of context names from the config"""
+    contexts = skypilot_config.get_effective_region_config(
+        cloud='kubernetes',
+        region=None,
+        keys=('context_configs',),
+        default_value={})
+    return [*contexts] or []
 
 
 # Mapping of known spot label keys and values for different cluster types
@@ -2602,6 +2625,21 @@ SPOT_LABEL_MAP = {
 }
 
 
+def get_autoscaler_type(
+    context: Optional[str] = None
+) -> Optional[kubernetes_enums.KubernetesAutoscalerType]:
+    """Returns the autoscaler type by reading from config"""
+    autoscaler_type = skypilot_config.get_effective_region_config(
+        cloud='kubernetes',
+        region=context,
+        keys=('autoscaler',),
+        default_value=None)
+    if autoscaler_type is not None:
+        autoscaler_type = kubernetes_enums.KubernetesAutoscalerType(
+            autoscaler_type)
+    return autoscaler_type
+
+
 def get_spot_label(
         context: Optional[str] = None) -> Tuple[Optional[str], Optional[str]]:
     """Get the spot label key and value for using spot instances, if supported.
@@ -2625,7 +2663,7 @@ def get_spot_label(
 
     # Check if autoscaler is configured. Allow spot instances if autoscaler type
     # is known to support spot instances.
-    autoscaler_type = get_autoscaler_type()
+    autoscaler_type = get_autoscaler_type(context=context)
     if autoscaler_type == kubernetes_enums.KubernetesAutoscalerType.GKE:
         return SPOT_LABEL_MAP[autoscaler_type.value]