skypilot-nightly 1.0.0.dev20250509__py3-none-any.whl → 1.0.0.dev20251107__py3-none-any.whl

sky/provision/hyperbolic/utils.py

@@ -0,0 +1,373 @@
+"""Hyperbolic API utilities."""
+import enum
+import json
+import os
+import time
+from typing import Any, Dict, Optional, Tuple
+
+import requests
+
+from sky import authentication
+from sky import sky_logging
+from sky.utils import status_lib
+
+#TODO update to prod endpoint
+BASE_URL = 'https://api.hyperbolic.xyz'
+API_KEY_PATH = '~/.hyperbolic/api_key'
+
+MAX_RETRIES = 3
+RETRY_DELAY = 2  # seconds
+TIMEOUT = 120
+
+logger = sky_logging.init_logger(__name__)
+
+
+class HyperbolicError(Exception):
+    """Base exception for Hyperbolic API errors."""
+    pass
+
+
+class HyperbolicInstanceStatus(enum.Enum):
+    """Statuses enum for Hyperbolic instances."""
+    UNKNOWN = 'unknown'
+    ONLINE = 'online'
+    OFFLINE = 'offline'
+    STARTING = 'starting'
+    STOPPING = 'stopping'
+    BUSY = 'busy'
+    RESTARTING = 'restarting'
+    CREATING = 'creating'
+    FAILED = 'failed'
+    ERROR = 'error'
+    TERMINATED = 'terminated'
+
+    @classmethod
+    def cluster_status_map(
+        cls
+    ) -> Dict['HyperbolicInstanceStatus', Optional[status_lib.ClusterStatus]]:
+        return {
+            cls.CREATING: status_lib.ClusterStatus.INIT,
+            cls.STARTING: status_lib.ClusterStatus.INIT,
+            cls.ONLINE: status_lib.ClusterStatus.UP,
+            cls.FAILED: status_lib.ClusterStatus.INIT,
+            cls.ERROR: status_lib.ClusterStatus.INIT,
+            cls.RESTARTING: status_lib.ClusterStatus.INIT,
+            cls.STOPPING: status_lib.ClusterStatus.INIT,
+            cls.UNKNOWN: status_lib.ClusterStatus.INIT,
+            cls.BUSY: status_lib.ClusterStatus.INIT,
+            cls.OFFLINE: status_lib.ClusterStatus.INIT,
+            cls.TERMINATED: None,
+        }
+
+    @classmethod
+    def from_raw_status(cls, status: str) -> 'HyperbolicInstanceStatus':
+        """Convert raw status string to HyperbolicInstanceStatus enum."""
+        try:
+            return cls(status.lower())
+        except ValueError as exc:
+            raise HyperbolicError(f'Unknown instance status: {status}') from exc
+
+    def to_cluster_status(self) -> Optional[status_lib.ClusterStatus]:
+        """Convert to SkyPilot cluster status."""
+        return self.cluster_status_map().get(self)
+
+
+class HyperbolicClient:
+    """Client for interacting with the Hyperbolic API."""
+
+    def __init__(self):
+        """Initialize the Hyperbolic client with API credentials."""
+        cred_path = os.path.expanduser(API_KEY_PATH)
+        if not os.path.exists(cred_path):
+            raise RuntimeError(f'API key not found at {cred_path}')
+        with open(cred_path, 'r', encoding='utf-8') as f:
+            self.api_key = f.read().strip()
+        self.headers = {'Authorization': f'Bearer {self.api_key}'}
+        self.api_url = BASE_URL
+
+    def _make_request(
+            self,
+            method: str,
+            endpoint: str,
+            payload: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+        """Make an API request to Hyperbolic."""
+        url = f'{BASE_URL}{endpoint}'
+        headers = {
+            'Authorization': f'Bearer {self.api_key}',
+            'Content-Type': 'application/json'
+        }
+
+        # Debug logging for request
+        logger.debug(f'Making {method} request to {url}')
+        if payload:
+            logger.debug(f'Request payload: {json.dumps(payload, indent=2)}')
+
+        try:
+            if method == 'GET':
+                response = requests.get(url, headers=headers, timeout=120)
+            elif method == 'POST':
+                response = requests.post(url,
+                                         headers=headers,
+                                         json=payload,
+                                         timeout=120)
+            else:
+                raise HyperbolicError(f'Unsupported HTTP method: {method}')
+
+            # Debug logging for response
+            logger.debug(f'Response status code: {response.status_code}')
+            logger.debug(f'Response headers: {dict(response.headers)}')
+
+            # Try to parse response as JSON
+            try:
+                response_data = response.json()
+                logger.debug(
+                    f'Response body: {json.dumps(response_data, indent=2)}')
+            except json.JSONDecodeError as exc:
+                # If response is not JSON, use the raw text
+                response_text = response.text
+                logger.debug(f'Response body (raw): {response_text}')
+                if not response.ok:
+                    raise HyperbolicError(f'API request failed with status '
+                                          f'{response.status_code}: '
+                                          f'{response_text}') from exc
+                # If response is OK but not JSON, return empty dict
+                return {}
+
+            if not response.ok:
+                error_msg = response_data.get(
+                    'error', response_data.get('message', response.text))
+                raise HyperbolicError(
+                    f'API request failed with status {response.status_code}: '
+                    f'{error_msg}')
+
+            return response_data
+        except requests.exceptions.RequestException as e:
+            raise HyperbolicError(f'Request failed: {str(e)}') from e
+        except Exception as e:
+            raise HyperbolicError(
+                f'Unexpected error during API request: {str(e)}') from e
+
+    def launch_instance(self, gpu_model: str, gpu_count: int,
+                        name: str) -> Tuple[str, str]:
+        """Launch a new instance with the specified configuration."""
+        # Initialize config with basic instance info
+        config = {
+            'gpuModel': gpu_model,
+            'gpuCount': str(gpu_count),
+            'userMetadata': {
+                'skypilot': {
+                    'cluster_name': name,
+                    'launch_time': str(int(time.time()))
+                }
+            }
+        }
+
+        config = authentication.setup_hyperbolic_authentication(config)
+
+        endpoint = '/v2/marketplace/instances/create-cheapest'
+        try:
+            response = self._make_request('POST', endpoint, payload=config)
+            logger.debug(f'Launch response: {json.dumps(response, indent=2)}')
+
+            instance_id = response.get('instanceName')
+            if not instance_id:
+                logger.error(f'No instance ID in response: {response}')
+                raise HyperbolicError('No instance ID returned from API')
+
+            logger.info(f'Successfully launched instance {instance_id}, '
+                        f'waiting for it to be ready...')
+
+            # Wait for instance to be ready
+            if not self.wait_for_instance(
+                    instance_id, HyperbolicInstanceStatus.ONLINE.value):
+                raise HyperbolicError(
+                    f'Instance {instance_id} failed to reach ONLINE state')
+
+            # Get instance details to get SSH command
+            instances = self.list_instances(
+                metadata={'skypilot': {
+                    'cluster_name': name
+                }})
+            instance = instances.get(instance_id)
+            if not instance:
+                raise HyperbolicError(
+                    f'Instance {instance_id} not found after launch')
+
+            ssh_command = instance.get('sshCommand')
+            if not ssh_command:
+                logger.error(
+                    f'No SSH command available for instance {instance_id}')
+                raise HyperbolicError('No SSH command available for instance')
+
+            logger.info(f'Instance {instance_id} is ready with SSH command')
+            return instance_id, ssh_command
+
+        except Exception as e:
+            logger.error(f'Failed to launch instance: {str(e)}')
+            raise HyperbolicError(f'Failed to launch instance: {str(e)}') from e
+
+    def list_instances(
+        self,
+        status: Optional[str] = None,
+        metadata: Optional[Dict[str, Dict[str, str]]] = None
+    ) -> Dict[str, Dict[str, Any]]:
+        """List all instances, optionally filtered by status and metadata."""
+        endpoint = '/v1/marketplace/instances'
+        try:
+            response = self._make_request('GET', endpoint)
+            logger.debug(f'Raw API response: {json.dumps(response, indent=2)}')
+            instances = {}
+            for instance in response.get('instances', []):
+                instance_info = instance.get('instance', {})
+                current_status = instance_info.get('status')
+                logger.debug(
+                    f'Instance {instance.get("id")} status: {current_status}')
+
+                # Convert raw status to enum
+                try:
+                    instance_status = HyperbolicInstanceStatus.from_raw_status(
+                        current_status)
+                except HyperbolicError as e:
+                    logger.warning(f'Failed to parse status for instance '
+                                   f'{instance.get("id")}: {e}')
+                    continue
+
+                if status and instance_status.value != status.lower():
+                    continue
+
+                if metadata:
+                    skypilot_metadata: Dict[str,
+                                            str] = metadata.get('skypilot', {})
+                    cluster_name = skypilot_metadata.get('cluster_name', '')
+                    instance_skypilot = instance.get('userMetadata',
+                                                     {}).get('skypilot', {})
+                    if not instance_skypilot.get('cluster_name',
+                                                 '').startswith(cluster_name):
+                        logger.debug(
+                            f'Skipping instance {instance.get("id")} - '
+                            f'skypilot metadata {instance_skypilot} '
+                            f'does not match {skypilot_metadata}')
+                        continue
+                    logger.debug(f'Including instance {instance.get("id")} '
+                                 f'- skypilot metadata matches')
+
+                hardware = instance_info.get('hardware', {})
+                instances[instance.get('id')] = {
+                    'id': instance.get('id'),
+                    'created': instance.get('created'),
+                    'sshCommand': instance.get('sshCommand'),
+                    'status': instance_status.value,
+                    'gpu_count': instance_info.get('gpu_count'),
+                    'gpus_total': instance_info.get('gpus_total'),
+                    'owner': instance_info.get('owner'),
+                    'cpus': hardware.get('cpus'),
+                    'gpus': hardware.get('gpus'),
+                    'ram': hardware.get('ram'),
+                    'storage': hardware.get('storage'),
+                    'pricing': instance_info.get('pricing'),
+                    'metadata': instance.get('userMetadata', {})
+                }
+            return instances
+        except Exception as e:
+            raise HyperbolicError(f'Failed to list instances: {str(e)}') from e
+
+    def terminate_instance(self, instance_id: str) -> None:
+        """Terminate an instance by ID."""
+        endpoint = '/v1/marketplace/instances/terminate'
+        data = {'id': instance_id}
+        try:
+            self._make_request('POST', endpoint, payload=data)
+        except Exception as e:
+            raise HyperbolicError(
+                f'Failed to terminate instance {instance_id}: {str(e)}') from e
+
+    def wait_for_instance(self,
+                          instance_id: str,
+                          target_status: str,
+                          timeout: int = TIMEOUT) -> bool:
+        """Wait for an instance to reach a specific status."""
+        start_time = time.time()
+        target_status_enum = HyperbolicInstanceStatus.from_raw_status(
+            target_status)
+        logger.info(
+            f'Waiting for instance {instance_id} '
+            f'to reach status {target_status_enum.value} and have SSH command')
+
+        while True:
+            elapsed = time.time() - start_time
+            if elapsed >= timeout:
+                logger.error(f'Timeout after {int(elapsed)}s '
+                             f'waiting for instance {instance_id}')
+                return False
+
+            try:
+                instances = self.list_instances()
+                instance = instances.get(instance_id)
+
+                if not instance:
+                    logger.warning(f'Instance {instance_id} not found')
+                    time.sleep(5)
+                    continue
+
+                current_status = instance.get('status', '').lower()
+                ssh_command = instance.get('sshCommand')
+                logger.debug(f'Current status: {current_status}, '
+                             f'Target status: {target_status_enum.value}, '
+                             f'SSH command: {ssh_command}')
+
+                if current_status == target_status_enum.value and ssh_command:
+                    logger.info(f'Instance {instance_id} reached '
+                                f'target status {target_status_enum.value} '
+                                f'and has SSH command after {int(elapsed)}s')
+                    return True
+
+                if current_status in ['failed', 'error', 'terminated']:
+                    logger.error(f'Instance {instance_id} reached '
+                                 f'terminal status: {current_status} '
+                                 f'after {int(elapsed)}s')
+                    return False
+
+                time.sleep(5)
+            except Exception as e:  # pylint: disable=broad-except
+                logger.warning(
+                    f'Error while waiting for instance {instance_id}: {str(e)}')
+                time.sleep(5)
+
+
+# Module-level singleton client
+_client = None
+
+
+def get_client() -> HyperbolicClient:
+    """Get or create the Hyperbolic client singleton."""
+    global _client
+    if _client is None:
+        _client = HyperbolicClient()
+    return _client
+
+
+# Backward-compatible wrapper functions
+def launch_instance(gpu_model: str, gpu_count: int,
+                    name: str) -> Tuple[str, str]:
+    """Launch a new instance with the specified configuration."""
+    return get_client().launch_instance(gpu_model, gpu_count, name)
+
+
+def list_instances(
+    status: Optional[str] = None,
+    metadata: Optional[Dict[str, Dict[str, str]]] = None
+) -> Dict[str, Dict[str, Any]]:
+    """List all instances, optionally filtered by status and metadata."""
+    return get_client().list_instances(status=status, metadata=metadata)
+
+
+def terminate_instance(instance_id: str) -> None:
+    """Terminate an instance by ID."""
+    return get_client().terminate_instance(instance_id)
+
+
+def wait_for_instance(instance_id: str,
+                      target_status: str,
+                      timeout: int = TIMEOUT) -> bool:
+    """Wait for an instance to reach a specific status."""
+    return get_client().wait_for_instance(instance_id, target_status, timeout)

sky/provision/instance_setup.py

@@ -8,7 +8,9 @@ import time
 from typing import Any, Callable, Dict, List, Optional, Tuple
 
 from sky import exceptions
+from sky import logs
 from sky import provision
+from sky import resources as resources_lib
 from sky import sky_logging
 from sky.provision import common
 from sky.provision import docker_utils
@@ -21,6 +23,7 @@ from sky.utils import accelerator_registry
 from sky.utils import command_runner
 from sky.utils import common_utils
 from sky.utils import env_options
+from sky.utils import resources_utils
 from sky.utils import subprocess_utils
 from sky.utils import timeline
 from sky.utils import ux_utils
@@ -82,7 +85,7 @@ def _set_usage_run_id_cmd() -> str:
     latest one when the function is called.
     """
     return (
-        f'cat {usage_constants.USAGE_RUN_ID_FILE} || '
+        f'cat {usage_constants.USAGE_RUN_ID_FILE} 2> /dev/null || '
         # The run id is retrieved locally for the current run, so that the
         # remote cluster will be set with the same run id as the initial
         # launch operation.
@@ -90,12 +93,6 @@ def _set_usage_run_id_cmd() -> str:
         f'{usage_constants.USAGE_RUN_ID_FILE}')
 
 
-def _set_skypilot_env_var_cmd() -> str:
-    """Sets the skypilot environment variables on the remote machine."""
-    env_vars = env_options.Options.all_options()
-    return '; '.join([f'export {k}={v}' for k, v in env_vars.items()])
-
-
 def _auto_retry(should_retry: Callable[[Exception], bool] = lambda _: True):
     """Decorator that retries the function if it fails.
 
@@ -134,6 +131,20 @@ def _hint_worker_log_path(cluster_name: str, cluster_info: common.ClusterInfo,
         logger.info(f'Logs of worker nodes can be found at: {worker_log_path}')
 
 
+class SSHThreadPoolExecutor(futures.ThreadPoolExecutor):
+    """ThreadPoolExecutor that kills children processes on exit."""
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        # ssh command runner eventually calls
+        # log_lib.run_with_log, which will spawn
+        # subprocesses. If we are exiting the context
+        # we need to kill the children processes
+        # to avoid leakage.
+        subprocess_utils.kill_children_processes()
+        self.shutdown()
+        return False
+
+
 def _parallel_ssh_with_cache(func,
                              cluster_name: str,
                              stage_name: str,
@@ -146,7 +157,7 @@ def _parallel_ssh_with_cache(func,
         # as 32 is too large for some machines.
         max_workers = subprocess_utils.get_parallel_threads(
             cluster_info.provider_name)
-    with futures.ThreadPoolExecutor(max_workers=max_workers) as pool:
+    with SSHThreadPoolExecutor(max_workers=max_workers) as pool:
         results = []
         runners = provision.get_command_runners(cluster_info.provider_name,
                                                 cluster_info, **ssh_credentials)
@@ -423,8 +434,16 @@ def start_ray_on_worker_nodes(cluster_name: str, no_restart: bool,
         # use the external IP of the head node.
         use_external_ip = cluster_info.custom_ray_options.pop(
             'use_external_ip', False)
-    head_ip = (head_instance.internal_ip
-               if not use_external_ip else head_instance.external_ip)
+
+    if use_external_ip:
+        head_ip = head_instance.external_ip
+    else:
+        # For Kubernetes, use the internal service address of the head node.
+        # Keep this consistent with the logic in kubernetes-ray.yml.j2
+        if head_instance.internal_svc:
+            head_ip = head_instance.internal_svc
+        else:
+            head_ip = head_instance.internal_ip
 
     ray_cmd = ray_worker_start_command(custom_resource,
                                        cluster_info.custom_ray_options,
@@ -466,11 +485,38 @@ def start_ray_on_worker_nodes(cluster_name: str, no_restart: bool,
 @common.log_function_start_end
 @_auto_retry()
 @timeline.event
-def start_skylet_on_head_node(cluster_name: str,
-                              cluster_info: common.ClusterInfo,
-                              ssh_credentials: Dict[str, Any]) -> None:
+def start_skylet_on_head_node(
+        cluster_name: resources_utils.ClusterName,
+        cluster_info: common.ClusterInfo, ssh_credentials: Dict[str, Any],
+        launched_resources: resources_lib.Resources) -> None:
     """Start skylet on the head node."""
-    del cluster_name
+    # Avoid circular import.
+    # pylint: disable=import-outside-toplevel
+    from sky.utils import controller_utils
+
+    def _set_skypilot_env_var_cmd() -> str:
+        """Sets the skypilot environment variables on the remote machine."""
+        env_vars = {
+            k: str(v) for (k, v) in env_options.Options.all_options().items()
+        }
+        is_controller = controller_utils.Controllers.from_name(
+            cluster_name.display_name) is not None
+        is_kubernetes = cluster_info.provider_name == 'kubernetes'
+        if is_controller and is_kubernetes:
+            # For jobs/serve controller, we pass in the CPU and memory limits
+            # when starting the skylet to handle cases where these env vars
+            # are not set on the cluster's pod spec. The skylet will read
+            # these env vars when starting (ManagedJobEvent.start()) and write
+            # it to disk.
+            resources = launched_resources.assert_launchable()
+            vcpus, mem = resources.cloud.get_vcpus_mem_from_instance_type(
+                resources.instance_type)
+            if vcpus is not None:
+                env_vars['SKYPILOT_POD_CPU_CORE_LIMIT'] = str(vcpus)
+            if mem is not None:
+                env_vars['SKYPILOT_POD_MEMORY_GB_LIMIT'] = str(mem)
+        return '; '.join([f'export {k}={v}' for k, v in env_vars.items()])
+
     runners = provision.get_command_runners(cluster_info.provider_name,
                                             cluster_info, **ssh_credentials)
     head_runner = runners[0]
@@ -557,3 +603,36 @@ def internal_file_mounts(cluster_name: str, common_file_mounts: Dict[str, str],
         ssh_credentials=ssh_credentials,
         max_workers=subprocess_utils.get_max_workers_for_file_mounts(
             common_file_mounts, cluster_info.provider_name))
+
+
+@common.log_function_start_end
+@timeline.event
+def setup_logging_on_cluster(logging_agent: logs.LoggingAgent,
+                             cluster_name: resources_utils.ClusterName,
+                             cluster_info: common.ClusterInfo,
+                             ssh_credentials: Dict[str, Any]) -> None:
+    """Setup logging agent (fluentbit) on all nodes after provisioning."""
+    _hint_worker_log_path(cluster_name.name_on_cloud, cluster_info,
+                          'logging_setup')
+
+    @_auto_retry()
+    def _setup_node(runner: command_runner.CommandRunner, log_path: str):
+        cmd = logging_agent.get_setup_command(cluster_name)
+        logger.info(f'Running command on node: {cmd}')
+        returncode, stdout, stderr = runner.run(cmd,
+                                                stream_logs=False,
+                                                require_outputs=True,
+                                                log_path=log_path,
+                                                source_bashrc=True)
+        if returncode:
+            raise RuntimeError(f'Failed to setup logging agent\n{cmd}\n'
+                               f'(exit code {returncode}). Error: '
+                               f'===== stdout ===== \n{stdout}\n'
+                               f'===== stderr ====={stderr}')
+
+    _parallel_ssh_with_cache(_setup_node,
+                             cluster_name.name_on_cloud,
+                             stage_name='logging_setup',
+                             digest=None,
+                             cluster_info=cluster_info,
+                             ssh_credentials=ssh_credentials)

sky/provision/kubernetes/__init__.py

@@ -11,3 +11,8 @@ from sky.provision.kubernetes.instance import wait_instances
 from sky.provision.kubernetes.network import cleanup_ports
 from sky.provision.kubernetes.network import open_ports
 from sky.provision.kubernetes.network import query_ports
+from sky.provision.kubernetes.volume import apply_volume
+from sky.provision.kubernetes.volume import delete_volume
+from sky.provision.kubernetes.volume import get_all_volumes_usedby
+from sky.provision.kubernetes.volume import get_volume_usedby
+from sky.provision.kubernetes.volume import map_all_volumes_usedby

sky/provision/kubernetes/config.py

@@ -3,20 +3,12 @@ import copy
 import logging
 import math
 import os
-import typing
-from typing import Any, Dict, Optional, Union
+from typing import Any, Dict, List, Optional, Union
 
-from sky.adaptors import common as adaptors_common
 from sky.adaptors import kubernetes
 from sky.provision import common
-from sky.provision.kubernetes import network_utils
 from sky.provision.kubernetes import utils as kubernetes_utils
-from sky.utils import kubernetes_enums
-
-if typing.TYPE_CHECKING:
-    import yaml
-else:
-    yaml = adaptors_common.LazyImport('yaml')
+from sky.utils import yaml_utils
 
 logger = logging.getLogger(__name__)
 
@@ -34,11 +26,6 @@ def bootstrap_instances(
 
     _configure_services(namespace, context, config.provider_config)
 
-    networking_mode = network_utils.get_networking_mode(
-        config.provider_config.get('networking_mode'))
-    if networking_mode == kubernetes_enums.KubernetesNetworkingMode.NODEPORT:
-        config = _configure_ssh_jump(namespace, context, config)
-
     requested_service_account = config.node_config['spec']['serviceAccountName']
     if (requested_service_account ==
             kubernetes_utils.DEFAULT_SERVICE_ACCOUNT_NAME):
@@ -487,41 +474,6 @@ def _configure_autoscaler_cluster_role_binding(
                 f'{created_msg(binding_field, name)}')
 
 
-def _configure_ssh_jump(namespace, context, config: common.ProvisionConfig):
-    """Creates a SSH jump pod to connect to the cluster.
-
-    Also updates config['auth']['ssh_proxy_command'] to use the newly created
-    jump pod.
-    """
-    provider_config = config.provider_config
-    pod_cfg = config.node_config
-
-    ssh_jump_name = pod_cfg['metadata']['labels']['skypilot-ssh-jump']
-    ssh_jump_image = provider_config['ssh_jump_image']
-
-    volumes = pod_cfg['spec']['volumes']
-    # find 'secret-volume' and get the secret name
-    secret_volume = next(filter(lambda x: x['name'] == 'secret-volume',
-                                volumes))
-    ssh_key_secret_name = secret_volume['secret']['secretName']
-
-    # TODO(romilb): We currently split SSH jump pod and svc creation. Service
-    #  is first created in authentication.py::setup_kubernetes_authentication
-    #  and then SSH jump pod creation happens here. This is because we need to
-    #  set the ssh_proxy_command in the ray YAML before we pass it to the
-    #  autoscaler. If in the future if we can write the ssh_proxy_command to the
-    #  cluster yaml through this method, then we should move the service
-    #  creation here.
-
-    # TODO(romilb): We should add a check here to make sure the service is up
-    #  and available before we create the SSH jump pod. If for any reason the
-    #  service is missing, we should raise an error.
-
-    kubernetes_utils.setup_ssh_jump_pod(ssh_jump_name, ssh_jump_image,
-                                        ssh_key_secret_name, namespace, context)
-    return config
-
-
 def _configure_skypilot_system_namespace(
         provider_config: Dict[str, Any]) -> None:
     """Creates the namespace for skypilot-system mounting if it does not exist.
@@ -592,7 +544,7 @@ def _configure_fuse_mounting(provider_config: Dict[str, Any]) -> None:
     daemonset_path = os.path.join(
         root_dir, 'kubernetes/manifests/fusermount-server-daemonset.yaml')
     with open(daemonset_path, 'r', encoding='utf-8') as file:
-        daemonset = yaml.safe_load(file)
+        daemonset = yaml_utils.safe_load(file)
     kubernetes_utils.merge_custom_metadata(daemonset['metadata'])
     try:
         kubernetes.apps_api(context).create_namespaced_daemon_set(
@@ -672,4 +624,9 @@ def _configure_services(namespace: str, context: Optional[str],
 
 
 class KubernetesError(Exception):
-    pass
+
+    def __init__(self,
+                 *args,
+                 insufficent_resources: Optional[List[str]] = None):
+        self.insufficent_resources = insufficent_resources
+        super().__init__(*args)

sky/provision/kubernetes/constants.py

@@ -6,3 +6,20 @@ NO_GPU_HELP_MESSAGE = ('If your cluster contains GPUs, make sure '
                        '(e.g., skypilot.co/accelerator) are setup correctly. ')
 
 KUBERNETES_IN_CLUSTER_NAMESPACE_ENV_VAR = 'SKYPILOT_IN_CLUSTER_NAMESPACE'
+
+# Name of kubernetes exec auth wrapper script
+SKY_K8S_EXEC_AUTH_WRAPPER = 'sky-kube-exec-wrapper'
+
+# PATH envvar for kubectl exec auth execve
+SKY_K8S_EXEC_AUTH_PATH = '$HOME/skypilot-runtime/bin:$HOME/google-cloud-sdk/bin:$PATH'  # pylint: disable=line-too-long
+
+# cache directory for kubeconfig with modified exec auth
+SKY_K8S_EXEC_AUTH_KUBECONFIG_CACHE = '~/.sky/generated/kubeconfigs'
+
+# Labels for the Pods created by SkyPilot
+TAG_RAY_CLUSTER_NAME = 'ray-cluster-name'
+TAG_POD_INITIALIZED = 'skypilot-initialized'
+TAG_SKYPILOT_DEPLOYMENT_NAME = 'skypilot-deployment-name'
+
+# Pod phases that are not holding PVCs
+PVC_NOT_HOLD_POD_PHASES = ['Succeeded', 'Failed']