skypilot-nightly 1.0.0.dev20250626__py3-none-any.whl → 1.0.0.dev20250628__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '512c53be5771c003cd9d94b5b89c23638a5e735e'
+_SKYPILOT_COMMIT_SHA = 'ba333007314473d8662d29933dca71a5e3d1fbf1'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250626'
+__version__ = '1.0.0.dev20250628'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

sky/adaptors/kubernetes.py

@@ -211,6 +211,13 @@ def api_client(context: Optional[str] = None):
     return kubernetes.client.ApiClient()
 
 
+@_api_logging_decorator('urllib3', logging.ERROR)
+@annotations.lru_cache(scope='request')
+def custom_resources_api(context: Optional[str] = None):
+    _load_config(context)
+    return kubernetes.client.CustomObjectsApi()
+
+
 @_api_logging_decorator('urllib3', logging.ERROR)
 @annotations.lru_cache(scope='request')
 def watch(context: Optional[str] = None):

sky/adaptors/nebius.py

@@ -120,8 +120,8 @@ def get_tenant_id():
         'tenant_id', None)
     if tenant_id_in_ws_config is not None:
         return tenant_id_in_ws_config
-    tenant_id_in_config = skypilot_config.get_nested(('nebius', 'tenant_id'),
-                                                     None)
+    tenant_id_in_config = skypilot_config.get_effective_region_config(
+        cloud='nebius', region=None, keys=('tenant_id',), default_value=None)
     if tenant_id_in_config is not None:
         return tenant_id_in_config
     try:

sky/admin_policy.py

@@ -2,7 +2,7 @@
 import abc
 import dataclasses
 import typing
-from typing import Any, Dict, Optional
+from typing import Optional
 
 import colorama
 import pydantic
@@ -44,8 +44,10 @@ class RequestOptions(pydantic.BaseModel):
 
 class _UserRequestBody(pydantic.BaseModel):
     """Auxiliary model to validate and serialize a user request."""
-    task: Dict[str, Any]
-    skypilot_config: Dict[str, Any]
+    # We have to use serialized YAML string, instead of a dict, because dict
+    # will be converted to JSON string, which will lose the None key.
+    task: str
+    skypilot_config: str
     request_options: Optional[RequestOptions] = None
     at_client_side: bool = False
 
@@ -78,25 +80,31 @@ class UserRequest:
 
     def encode(self) -> str:
         return _UserRequestBody(
-            task=self.task.to_yaml_config(),
-            skypilot_config=dict(self.skypilot_config),
+            task=common_utils.dump_yaml_str(self.task.to_yaml_config()),
+            skypilot_config=common_utils.dump_yaml_str(
+                dict(self.skypilot_config)),
             request_options=self.request_options,
-            at_client_side=self.at_client_side).model_dump_json()
+            at_client_side=self.at_client_side,
+        ).model_dump_json()
 
     @classmethod
     def decode(cls, body: str) -> 'UserRequest':
         user_request_body = _UserRequestBody.model_validate_json(body)
-        return cls(task=sky.Task.from_yaml_config(user_request_body.task),
-                   skypilot_config=config_utils.Config.from_dict(
-                       user_request_body.skypilot_config),
-                   request_options=user_request_body.request_options,
-                   at_client_side=user_request_body.at_client_side)
+        return cls(
+            task=sky.Task.from_yaml_config(
+                common_utils.read_yaml_all_str(user_request_body.task)[0]),
+            skypilot_config=config_utils.Config.from_dict(
+                common_utils.read_yaml_all_str(
+                    user_request_body.skypilot_config)[0]),
+            request_options=user_request_body.request_options,
+            at_client_side=user_request_body.at_client_side,
+        )
 
 
 class _MutatedUserRequestBody(pydantic.BaseModel):
     """Auxiliary model to validate and serialize a user request."""
-    task: Dict[str, Any]
-    skypilot_config: Dict[str, Any]
+    task: str
+    skypilot_config: str
 
 
 @dataclasses.dataclass
@@ -108,17 +116,19 @@ class MutatedUserRequest:
 
     def encode(self) -> str:
         return _MutatedUserRequestBody(
-            task=self.task.to_yaml_config(),
-            skypilot_config=dict(self.skypilot_config)).model_dump_json()
+            task=common_utils.dump_yaml_str(self.task.to_yaml_config()),
+            skypilot_config=common_utils.dump_yaml_str(
+                dict(self.skypilot_config),)).model_dump_json()
 
     @classmethod
     def decode(cls, mutated_user_request_body: str) -> 'MutatedUserRequest':
         mutated_user_request_body = _MutatedUserRequestBody.model_validate_json(
             mutated_user_request_body)
         return cls(task=sky.Task.from_yaml_config(
-            mutated_user_request_body.task),
+            common_utils.read_yaml_all_str(mutated_user_request_body.task)[0]),
                    skypilot_config=config_utils.Config.from_dict(
-                       mutated_user_request_body.skypilot_config))
+                       common_utils.read_yaml_all_str(
+                           mutated_user_request_body.skypilot_config)[0],))
 
 
 class PolicyInterface:

sky/authentication.py

@@ -419,12 +419,17 @@ def setup_ibm_authentication(config: Dict[str, Any]) -> Dict[str, Any]:
 
 
 def setup_kubernetes_authentication(config: Dict[str, Any]) -> Dict[str, Any]:
+    context = kubernetes_utils.get_context_from_config(config['provider'])
+
     # Default ssh session is established with kubectl port-forwarding with
     # ClusterIP service.
     nodeport_mode = kubernetes_enums.KubernetesNetworkingMode.NODEPORT
     port_forward_mode = kubernetes_enums.KubernetesNetworkingMode.PORTFORWARD
-    network_mode_str = skypilot_config.get_nested(('kubernetes', 'networking'),
-                                                  port_forward_mode.value)
+    network_mode_str = skypilot_config.get_effective_region_config(
+        cloud='kubernetes',
+        region=context,
+        keys=('networking',),
+        default_value=port_forward_mode.value)
     try:
         network_mode = kubernetes_enums.KubernetesNetworkingMode.from_str(
             network_mode_str)
@@ -439,7 +444,6 @@ def setup_kubernetes_authentication(config: Dict[str, Any]) -> Dict[str, Any]:
     # Add the user's public key to the SkyPilot cluster.
     secret_name = clouds.Kubernetes.SKY_SSH_KEY_SECRET_NAME
     secret_field_name = clouds.Kubernetes().ssh_key_secret_field_name
-    context = kubernetes_utils.get_context_from_config(config['provider'])
     namespace = kubernetes_utils.get_namespace_from_config(config['provider'])
     k8s = kubernetes.kubernetes
     with open(public_key_path, 'r', encoding='utf-8') as f:
@@ -454,8 +458,11 @@ def setup_kubernetes_authentication(config: Dict[str, Any]) -> Dict[str, Any]:
                 'parent': 'skypilot'
             }
         }
-        custom_metadata = skypilot_config.get_nested(
-            ('kubernetes', 'custom_metadata'), {})
+        custom_metadata = skypilot_config.get_effective_region_config(
+            cloud='kubernetes',
+            region=context,
+            keys=('custom_metadata',),
+            default_value={})
         config_utils.merge_k8s_configs(secret_metadata, custom_metadata)
 
         secret = k8s.client.V1Secret(

sky/backends/backend_utils.py

@@ -520,11 +520,51 @@ def get_expirable_clouds(
     expirable_clouds = []
     local_credentials_value = schemas.RemoteIdentityOptions.LOCAL_CREDENTIALS.value
     for cloud in enabled_clouds:
-        remote_identities = skypilot_config.get_nested(
-            (str(cloud).lower(), 'remote_identity'), None)
-        if remote_identities is None:
-            remote_identities = schemas.get_default_remote_identity(
-                str(cloud).lower())
+        # Kubernetes config might have context-specific properties
+        if isinstance(cloud, clouds.Kubernetes):
+            # get all custom contexts
+            contexts = kubernetes_utils.get_custom_config_k8s_contexts()
+            # add remote_identity of each context if it exists
+            remote_identities = None
+            for context in contexts:
+                context_remote_identity = skypilot_config.get_effective_region_config(
+                    cloud='kubernetes',
+                    region=context,
+                    keys=('remote_identity',),
+                    default_value=None)
+                if context_remote_identity is not None:
+                    if remote_identities is None:
+                        remote_identities = []
+                    if isinstance(context_remote_identity, str):
+                        remote_identities.append(
+                            {context: context_remote_identity})
+                    elif isinstance(context_remote_identity, list):
+                        remote_identities.extend(context_remote_identity)
+            # add global kubernetes remote identity if it exists, if not, add default
+            global_remote_identity = skypilot_config.get_effective_region_config(
+                cloud='kubernetes',
+                region=None,
+                keys=('remote_identity',),
+                default_value=None)
+            if global_remote_identity is not None:
+                if remote_identities is None:
+                    remote_identities = []
+                if isinstance(global_remote_identity, str):
+                    remote_identities.append({'*': global_remote_identity})
+                elif isinstance(global_remote_identity, list):
+                    remote_identities.extend(global_remote_identity)
+            if remote_identities is None:
+                remote_identities = schemas.get_default_remote_identity(
+                    str(cloud).lower())
+        else:
+            remote_identities = skypilot_config.get_effective_region_config(
+                cloud=str(cloud).lower(),
+                region=None,
+                keys=('remote_identity',),
+                default_value=None)
+            if remote_identities is None:
+                remote_identities = schemas.get_default_remote_identity(
+                    str(cloud).lower())
 
         local_credential_expiring = cloud.can_credential_expire()
         if isinstance(remote_identities, str):
@@ -605,8 +645,11 @@ def write_cluster_config(
     config_dict = {}
 
     specific_reservations = set(
-        skypilot_config.get_nested(
-            (str(to_provision.cloud).lower(), 'specific_reservations'), set()))
+        skypilot_config.get_effective_region_config(
+            cloud=str(to_provision.cloud).lower(),
+            region=to_provision.region,
+            keys=('specific_reservations',),
+            default_value=set()))
 
     # Remote identity handling can have 4 cases:
     # 1. LOCAL_CREDENTIALS (default for most clouds): Upload local credentials
@@ -619,8 +662,11 @@ def write_cluster_config(
     # running required checks.
     assert cluster_name is not None
     excluded_clouds: Set[clouds.Cloud] = set()
-    remote_identity_config = skypilot_config.get_nested(
-        (str(cloud).lower(), 'remote_identity'), None)
+    remote_identity_config = skypilot_config.get_effective_region_config(
+        cloud=str(cloud).lower(),
+        region=region.name,
+        keys=('remote_identity',),
+        default_value=None)
     remote_identity = schemas.get_default_remote_identity(str(cloud).lower())
     if isinstance(remote_identity_config, str):
         remote_identity = remote_identity_config
@@ -649,15 +695,21 @@ def write_cluster_config(
                 'is not supported by this cloud. Remove the config or set: '
                 '`remote_identity: LOCAL_CREDENTIALS`.')
         if isinstance(cloud, clouds.Kubernetes):
-            if skypilot_config.get_nested(
-                ('kubernetes', 'allowed_contexts'), None) is None:
+            if skypilot_config.get_effective_region_config(
+                    cloud='kubernetes',
+                    region=None,
+                    keys=('allowed_contexts',),
+                    default_value=None) is None:
                 excluded_clouds.add(cloud)
         else:
             excluded_clouds.add(cloud)
 
     for cloud_str, cloud_obj in registry.CLOUD_REGISTRY.items():
-        remote_identity_config = skypilot_config.get_nested(
-            (cloud_str.lower(), 'remote_identity'), None)
+        remote_identity_config = skypilot_config.get_effective_region_config(
+            cloud=cloud_str.lower(),
+            region=region.name,
+            keys=('remote_identity',),
+            default_value=None)
         if remote_identity_config:
             if (remote_identity_config ==
                     schemas.RemoteIdentityOptions.NO_UPLOAD.value):
@@ -678,8 +730,11 @@ def write_cluster_config(
     yaml_path = _get_yaml_path_from_cluster_name(cluster_name)
 
     # Retrieve the ssh_proxy_command for the given cloud / region.
-    ssh_proxy_command_config = skypilot_config.get_nested(
-        (str(cloud).lower(), 'ssh_proxy_command'), None)
+    ssh_proxy_command_config = skypilot_config.get_effective_region_config(
+        cloud=str(cloud).lower(),
+        region=None,
+        keys=('ssh_proxy_command',),
+        default_value=None)
     if (isinstance(ssh_proxy_command_config, str) or
             ssh_proxy_command_config is None):
         ssh_proxy_command = ssh_proxy_command_config
@@ -705,7 +760,11 @@ def write_cluster_config(
     logger.debug(f'Using ssh_proxy_command: {ssh_proxy_command!r}')
 
     # User-supplied global instance tags from ~/.sky/config.yaml.
-    labels = skypilot_config.get_nested((str(cloud).lower(), 'labels'), {})
+    labels = skypilot_config.get_effective_region_config(
+        cloud=str(cloud).lower(),
+        region=region.name,
+        keys=('labels',),
+        default_value={})
     # labels is a dict, which is guaranteed by the type check in
     # schemas.py
     assert isinstance(labels, dict), labels
@@ -762,12 +821,17 @@ def write_cluster_config(
                     os.environ.get(constants.USER_ENV_VAR, '')),
 
                 # Networking configs
-                'use_internal_ips': skypilot_config.get_nested(
-                    (str(cloud).lower(), 'use_internal_ips'), False),
+                'use_internal_ips': skypilot_config.get_effective_region_config(
+                    cloud=str(cloud).lower(),
+                    region=region.name,
+                    keys=('use_internal_ips',),
+                    default_value=False),
                 'ssh_proxy_command': ssh_proxy_command,
-                'vpc_name': skypilot_config.get_nested(
-                    (str(cloud).lower(), 'vpc_name'), None),
-
+                'vpc_name': skypilot_config.get_effective_region_config(
+                    cloud=str(cloud).lower(),
+                    region=region.name,
+                    keys=('vpc_name',),
+                    default_value=None),
                 # User-supplied labels.
                 'labels': labels,
                 # User-supplied remote_identity
@@ -847,8 +911,9 @@ def write_cluster_config(
         kubernetes_utils.combine_pod_config_fields(
             tmp_yaml_path,
             cluster_config_overrides=cluster_config_overrides,
-            cloud=cloud)
-        kubernetes_utils.combine_metadata_fields(tmp_yaml_path)
+            cloud=cloud,
+            context=region.name)
+        kubernetes_utils.combine_metadata_fields(tmp_yaml_path, region.name)
         yaml_obj = common_utils.read_yaml(tmp_yaml_path)
         pod_config: Dict[str, Any] = yaml_obj['available_node_types'][
             'ray_head_default']['node_config']
@@ -3095,7 +3160,8 @@ def get_endpoints(cluster: str,
                          f'{_ENDPOINTS_RETRY_MESSAGE} ')
             if launched_resources.cloud.is_same_cloud(clouds.Kubernetes()):
                 # Add Kubernetes specific debugging info
-                error_msg += (kubernetes_utils.get_endpoint_debug_message())
+                error_msg += kubernetes_utils.get_endpoint_debug_message(
+                    launched_resources.region)
             logger.warning(error_msg)
             return {}
         return {port: port_details[port][0].url()}
@@ -3113,8 +3179,8 @@ def get_endpoints(cluster: str,
                              f'{_ENDPOINTS_RETRY_MESSAGE} ')
                 if launched_resources.cloud.is_same_cloud(clouds.Kubernetes()):
                     # Add Kubernetes specific debugging info
-                    error_msg += \
-                        kubernetes_utils.get_endpoint_debug_message()
+                    error_msg += kubernetes_utils.get_endpoint_debug_message(
+                        launched_resources.region)
                 logger.warning(error_msg)
                 return {}
         return {

sky/check.py

@@ -619,8 +619,11 @@ def _format_enabled_cloud(cloud_name: str,
             return _green_color(cloud_and_capabilities)
 
         # Check if allowed_contexts is explicitly set in config
-        allowed_contexts = skypilot_config.get_nested(
-            ('kubernetes', 'allowed_contexts'), None)
+        allowed_contexts = skypilot_config.get_effective_region_config(
+            cloud='kubernetes',
+            region=None,
+            keys=('allowed_contexts',),
+            default_value=None)
 
         # Format the context info with consistent styling
         if allowed_contexts is not None:

sky/client/cli/command.py

@@ -3176,7 +3176,11 @@ def show_gpus(
         cloud_obj, clouds.Kubernetes) and not isinstance(cloud_obj, clouds.SSH)
     cloud_is_ssh = isinstance(cloud_obj, clouds.SSH)
     # TODO(romilb): We should move this to the backend.
-    kubernetes_autoscaling = kubernetes_utils.get_autoscaler_type() is not None
+    kubernetes_autoscaling = skypilot_config.get_effective_region_config(
+        cloud='kubernetes',
+        region=region,
+        keys=('autoscaler',),
+        default_value=None) is not None
     kubernetes_is_enabled = clouds.Kubernetes.canonical_name() in enabled_clouds
     ssh_is_enabled = clouds.SSH.canonical_name() in enabled_clouds
     query_k8s_realtime_gpu = (kubernetes_is_enabled and
@@ -5358,14 +5362,42 @@ def api_status(request_ids: Optional[List[str]], all_status: bool,
               '-e',
               required=False,
               help='The SkyPilot API server endpoint.')
-@click.option('--get-token',
+@click.option('--relogin',
               is_flag=True,
               default=False,
-              help='Force token-based login.')
+              help='Force relogin with OAuth2 when enabled.')
+@click.option(
+    '--service-account-token',
+    '--token',
+    '-t',
+    required=False,
+    help='Service account token for authentication (starts with ``sky_``).')
 @usage_lib.entrypoint
-def api_login(endpoint: Optional[str], get_token: bool):
-    """Logs into a SkyPilot API server."""
-    sdk.api_login(endpoint, get_token)
+def api_login(endpoint: Optional[str], relogin: bool,
+              service_account_token: Optional[str]):
+    """Logs into a SkyPilot API server.
+
+    If your remote API server has enabled OAuth2 authentication, you can use
+    one of the following methods to login:
+
+    1. OAuth2 browser-based authentication (default)
+
+    2. Service account token via ``--token`` flag
+
+    3. Service account token in ``~/.sky/config.yaml``
+
+    Examples:
+
+    .. code-block:: bash
+
+      # OAuth2 browser login
+      sky api login -e https://api.example.com
+      \b
+      # Service account token login
+      sky api login -e https://api.example.com --token sky_abc123...
+
+    """
+    sdk.api_login(endpoint, relogin, service_account_token)
 
 
 @api.command('info', cls=_DocumentedCodeCommand)