skypilot-nightly 1.0.0.dev20251012__py3-none-any.whl → 1.0.0.dev20251014__py3-none-any.whl

sky/__init__.py

@@ -7,7 +7,7 @@ import urllib.request
 from sky.utils import directory_utils
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '7d5d1a2925fc7192af10061ca395d329364e7405'
+_SKYPILOT_COMMIT_SHA = '7bcf0fcb2073aac435139d1d85fc0e66acca26a5'
 
 
 def _get_git_commit():
@@ -37,7 +37,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20251012'
+__version__ = '1.0.0.dev20251014'
 __root_dir__ = directory_utils.get_sky_dir()
 
 
@@ -150,6 +150,7 @@ Vsphere = clouds.Vsphere
 Fluidstack = clouds.Fluidstack
 Nebius = clouds.Nebius
 Hyperbolic = clouds.Hyperbolic
+Shadeform = clouds.Shadeform
 Seeweb = clouds.Seeweb
 
 __all__ = [
@@ -172,6 +173,7 @@ __all__ = [
     'Fluidstack',
     'Nebius',
     'Hyperbolic',
+    'Shadeform',
     'Seeweb',
     'Optimizer',
     'OptimizeTarget',

sky/adaptors/shadeform.py

@@ -0,0 +1,89 @@
+"""Shadeform cloud adaptor."""
+
+import functools
+import socket
+from typing import Any, Dict, List, Optional
+
+import requests
+
+from sky import sky_logging
+from sky.provision.shadeform import shadeform_utils
+from sky.utils import common_utils
+
+logger = sky_logging.init_logger(__name__)
+
+_shadeform_sdk = None
+
+
+def import_package(func):
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        global _shadeform_sdk
+        if _shadeform_sdk is None:
+            try:
+                import shadeform as _shadeform  # pylint: disable=import-outside-toplevel
+                _shadeform_sdk = _shadeform
+            except ImportError:
+                raise ImportError(
+                    'Failed to import dependencies for Shadeform. '
+                    'Try pip install "skypilot[shadeform]"') from None
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+@import_package
+def shadeform():
+    """Return the shadeform package."""
+    return _shadeform_sdk
+
+
+def list_ssh_keys() -> List[Dict[str, Any]]:
+    """List all SSH keys in Shadeform account."""
+    try:
+        response = shadeform_utils.get_ssh_keys()
+        return response.get('ssh_keys', [])
+    except (ValueError, KeyError, requests.exceptions.RequestException) as e:
+        logger.warning(f'Failed to list SSH keys from Shadeform: {e}')
+        return []
+
+
+def add_ssh_key_to_shadeform(public_key: str) -> Optional[str]:
+    """Add SSH key to Shadeform if it doesn't already exist.
+
+    Args:
+        public_key: The SSH public key string.
+
+    Returns:
+        The name of the key if added successfully, None otherwise.
+    """
+    try:
+        # Check if key already exists
+        existing_keys = list_ssh_keys()
+        key_exists = False
+        key_id = None
+        for key in existing_keys:
+            if key.get('public_key') == public_key:
+                key_exists = True
+                key_id = key.get('id')
+                break
+
+        if key_exists:
+            logger.info('SSH key already exists in Shadeform account')
+            return key_id
+
+        # Generate a unique key name
+        hostname = socket.gethostname()
+        key_name = f'skypilot-{hostname}-{common_utils.get_user_hash()[:8]}'
+
+        # Add the key
+        response = shadeform_utils.add_ssh_key(name=key_name,
+                                               public_key=public_key)
+        key_id = response['id']
+        logger.info(f'Added SSH key to Shadeform: {key_name, key_id}')
+        return key_id
+
+    except (ValueError, KeyError, requests.exceptions.RequestException) as e:
+        logger.warning(f'Failed to add SSH key to Shadeform: {e}')
+        return None

sky/authentication.py

@@ -39,6 +39,7 @@ from sky.adaptors import gcp
 from sky.adaptors import ibm
 from sky.adaptors import runpod
 from sky.adaptors import seeweb as seeweb_adaptor
+from sky.adaptors import shadeform as shadeform_adaptor
 from sky.adaptors import vast
 from sky.provision.fluidstack import fluidstack_utils
 from sky.provision.kubernetes import utils as kubernetes_utils
@@ -152,7 +153,12 @@ def get_or_generate_keys() -> Tuple[str, str]:
     return private_key_path, public_key_path
 
 
-def create_ssh_key_files_from_db(private_key_path: str):
+def create_ssh_key_files_from_db(private_key_path: str) -> bool:
+    """Creates the ssh key files from the database.
+
+    Returns:
+        True if the ssh key files are created successfully, False otherwise.
+    """
     # Assume private key path is in the format of
     # ~/.sky/clients/<user_hash>/ssh/sky-key
     separated_path = os.path.normpath(private_key_path).split(os.path.sep)
@@ -180,12 +186,14 @@ def create_ssh_key_files_from_db(private_key_path: str):
             ssh_public_key, ssh_private_key, exists = (
                 global_user_state.get_ssh_keys(user_hash))
             if not exists:
-                raise RuntimeError(f'SSH keys not found for user {user_hash}')
+                logger.debug(f'SSH keys not found for user {user_hash}')
+                return False
             _save_key_pair(private_key_path, public_key_path, ssh_private_key,
                            ssh_public_key)
     assert os.path.exists(public_key_path), (
         'Private key found, but associated public key '
         f'{public_key_path} does not exist.')
+    return True
 
 
 def configure_ssh_info(config: Dict[str, Any]) -> Dict[str, Any]:
@@ -511,6 +519,48 @@ def setup_hyperbolic_authentication(config: Dict[str, Any]) -> Dict[str, Any]:
     return configure_ssh_info(config)
 
 
+def setup_shadeform_authentication(config: Dict[str, Any]) -> Dict[str, Any]:
+    """Sets up SSH authentication for Shadeform.
+    - Generates a new SSH key pair if one does not exist.
+    - Adds the public SSH key to the user's Shadeform account.
+
+    Note: This assumes there is a Shadeform Python SDK available.
+    If no official SDK exists, this function would need to use direct API calls.
+    """
+
+    _, public_key_path = get_or_generate_keys()
+    ssh_key_id = None
+
+    with open(public_key_path, 'r', encoding='utf-8') as f:
+        public_key = f.read().strip()
+
+    try:
+        # Add SSH key to Shadeform using our utility functions
+        ssh_key_id = shadeform_adaptor.add_ssh_key_to_shadeform(public_key)
+
+    except ImportError as e:
+        # If required dependencies are missing
+        logger.warning(
+            f'Failed to add Shadeform SSH key due to missing dependencies: '
+            f'{e}. Manually configure SSH keys in your Shadeform account.')
+
+    except Exception as e:
+        logger.warning(f'Failed to set up Shadeform authentication: {e}')
+        raise exceptions.CloudUserIdentityError(
+            'Failed to set up SSH authentication for Shadeform. '
+            f'Please ensure your Shadeform credentials are configured: {e}'
+        ) from e
+
+    if ssh_key_id is None:
+        raise Exception('Failed to add SSH key to Shadeform')
+
+    # Configure SSH info in the config
+    config['auth']['ssh_public_key'] = public_key_path
+    config['auth']['ssh_key_id'] = ssh_key_id
+
+    return configure_ssh_info(config)
+
+
 def setup_primeintellect_authentication(
         config: Dict[str, Any]) -> Dict[str, Any]:
     """Sets up SSH authentication for Prime Intellect.

sky/backends/backend_utils.py

@@ -1124,6 +1124,8 @@ def _add_auth_to_cluster_config(cloud: clouds.Cloud, tmp_yaml_path: str):
         config = auth.setup_fluidstack_authentication(config)
     elif isinstance(cloud, clouds.Hyperbolic):
         config = auth.setup_hyperbolic_authentication(config)
+    elif isinstance(cloud, clouds.Shadeform):
+        config = auth.setup_shadeform_authentication(config)
     elif isinstance(cloud, clouds.PrimeIntellect):
         config = auth.setup_primeintellect_authentication(config)
     elif isinstance(cloud, clouds.Seeweb):
@@ -1855,6 +1857,13 @@ def check_owner_identity(cluster_name: str) -> None:
                                                      summary_response=True)
     if record is None:
         return
+    _check_owner_identity_with_record(cluster_name, record)
+
+
+def _check_owner_identity_with_record(cluster_name: str,
+                                      record: Dict[str, Any]) -> None:
+    if env_options.Options.SKIP_CLOUD_IDENTITY_CHECK.get():
+        return
     handle = record['handle']
     if not isinstance(handle, backends.CloudVmRayResourceHandle):
         return
@@ -2149,6 +2158,7 @@ def check_can_clone_disk_and_override_task(
 
 def _update_cluster_status(
         cluster_name: str,
+        record: Dict[str, Any],
         include_user_info: bool = True,
         summary_response: bool = False) -> Optional[Dict[str, Any]]:
     """Update the cluster status.
@@ -2177,12 +2187,6 @@ def _update_cluster_status(
           fetched from the cloud provider or there are leaked nodes causing
           the node number larger than expected.
     """
-    record = global_user_state.get_cluster_from_name(
-        cluster_name,
-        include_user_info=include_user_info,
-        summary_response=summary_response)
-    if record is None:
-        return None
     handle = record['handle']
     if handle.cluster_yaml is None:
         # Remove cluster from db since this cluster does not have a config file
@@ -2675,10 +2679,9 @@ def refresh_cluster_record(
     # using the correct cloud credentials.
     workspace = record.get('workspace', constants.SKYPILOT_DEFAULT_WORKSPACE)
     with skypilot_config.local_active_workspace_ctx(workspace):
-        check_owner_identity(cluster_name)
-
-        if not isinstance(record['handle'], backends.CloudVmRayResourceHandle):
-            return record
+        # check_owner_identity returns if the record handle is
+        # not a CloudVmRayResourceHandle
+        _check_owner_identity_with_record(cluster_name, record)
 
         # The loop logic allows us to notice if the status was updated in the
         # global_user_state by another process and stop trying to get the lock.
@@ -2695,7 +2698,8 @@ def refresh_cluster_record(
                 return record
 
             if cluster_lock_already_held:
-                return _update_cluster_status(cluster_name, include_user_info,
+                return _update_cluster_status(cluster_name, record,
+                                              include_user_info,
                                               summary_response)
 
             # Try to acquire the lock so we can fetch the status.
@@ -2711,7 +2715,7 @@ def refresh_cluster_record(
                             record, force_refresh_statuses):
                         return record
                     # Update and return the cluster status.
-                    return _update_cluster_status(cluster_name,
+                    return _update_cluster_status(cluster_name, record,
                                                   include_user_info,
                                                   summary_response)
 
@@ -3115,25 +3119,23 @@ def refresh_cluster_records() -> None:
     exclude_managed_clusters = True
     if env_options.Options.SHOW_DEBUG_INFO.get():
         exclude_managed_clusters = False
-    cluster_names = global_user_state.get_cluster_names(
-        exclude_managed_clusters=exclude_managed_clusters,)
+    cluster_names = set(
+        global_user_state.get_cluster_names(
+            exclude_managed_clusters=exclude_managed_clusters,))
 
     # TODO(syang): we should try not to leak
     # request info in backend_utils.py.
     # Refactor this to use some other info to
     # determine if a launch is in progress.
-    request = requests_lib.get_request_tasks(
+    requests = requests_lib.get_request_tasks(
         req_filter=requests_lib.RequestTaskFilter(
             status=[requests_lib.RequestStatus.RUNNING],
-            cluster_names=cluster_names,
             include_request_names=['sky.launch']))
     cluster_names_with_launch_request = {
-        request.cluster_name for request in request
+        request.cluster_name for request in requests
     }
-    cluster_names_without_launch_request = [
-        cluster_name for cluster_name in cluster_names
-        if cluster_name not in cluster_names_with_launch_request
-    ]
+    cluster_names_without_launch_request = (cluster_names -
+                                            cluster_names_with_launch_request)
 
     def _refresh_cluster_record(cluster_name):
         return _refresh_cluster(cluster_name,
@@ -3142,7 +3144,7 @@ def refresh_cluster_records() -> None:
                                 include_user_info=False,
                                 summary_response=True)
 
-    if len(cluster_names) > 0:
+    if len(cluster_names_without_launch_request) > 0:
         # Do not refresh the clusters that have an active launch request.
         subprocess_utils.run_in_parallel(_refresh_cluster_record,
                                          cluster_names_without_launch_request)
@@ -3268,7 +3270,15 @@ def get_clusters(
                 expanded_private_key_path = os.path.expanduser(
                     ssh_private_key_path)
                 if not os.path.exists(expanded_private_key_path):
-                    auth.create_ssh_key_files_from_db(ssh_private_key_path)
+                    success = auth.create_ssh_key_files_from_db(
+                        ssh_private_key_path)
+                    if not success:
+                        # If the ssh key files are not found, we do not
+                        # update the record with credentials.
+                        logger.debug(
+                            f'SSH keys not found for cluster {record["name"]} '
+                            f'at key path {ssh_private_key_path}')
+                        continue
             else:
                 private_key_path, _ = auth.get_or_generate_keys()
                 expanded_private_key_path = os.path.expanduser(private_key_path)
@@ -3342,13 +3352,13 @@ def get_clusters(
     # request info in backend_utils.py.
     # Refactor this to use some other info to
     # determine if a launch is in progress.
-    request = requests_lib.get_request_tasks(
+    requests = requests_lib.get_request_tasks(
         req_filter=requests_lib.RequestTaskFilter(
             status=[requests_lib.RequestStatus.RUNNING],
             cluster_names=cluster_names,
             include_request_names=['sky.launch']))
     cluster_names_with_launch_request = {
-        request.cluster_name for request in request
+        request.cluster_name for request in requests
     }
     cluster_names_without_launch_request = [
         cluster_name for cluster_name in cluster_names

sky/backends/cloud_vm_ray_backend.py

@@ -141,6 +141,7 @@ _NODES_LAUNCHING_PROGRESS_TIMEOUT = {
     clouds.OCI: 300,
     clouds.Paperspace: 600,
     clouds.Kubernetes: 300,
+    clouds.Shadeform: 300,
     clouds.Vsphere: 240,
 }
 
@@ -304,6 +305,7 @@ def _get_cluster_config_template(cloud):
         clouds.RunPod: 'runpod-ray.yml.j2',
         clouds.Kubernetes: 'kubernetes-ray.yml.j2',
         clouds.SSH: 'kubernetes-ray.yml.j2',
+        clouds.Shadeform: 'shadeform-ray.yml.j2',
         clouds.Vsphere: 'vsphere-ray.yml.j2',
         clouds.Vast: 'vast-ray.yml.j2',
         clouds.Fluidstack: 'fluidstack-ray.yml.j2',
@@ -3718,7 +3720,7 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
 
                 self._update_after_cluster_provisioned(
                     handle, to_provision_config.prev_handle, task,
-                    prev_cluster_status, lock_id, config_hash)
+                    prev_cluster_status, config_hash)
                 return handle, False
 
             cluster_config_file = config_dict['ray']
@@ -3790,7 +3792,7 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
 
             self._update_after_cluster_provisioned(
                 handle, to_provision_config.prev_handle, task,
-                prev_cluster_status, lock_id, config_hash)
+                prev_cluster_status, config_hash)
             return handle, False
 
     def _open_ports(self, handle: CloudVmRayResourceHandle) -> None:
@@ -3808,7 +3810,7 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
             prev_handle: Optional[CloudVmRayResourceHandle],
             task: task_lib.Task,
             prev_cluster_status: Optional[status_lib.ClusterStatus],
-            lock_id: str, config_hash: str) -> None:
+            config_hash: str) -> None:
         usage_lib.messages.usage.update_cluster_resources(
             handle.launched_nodes, handle.launched_resources)
         usage_lib.messages.usage.update_final_cluster_status(
@@ -3920,8 +3922,6 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
                 handle.cached_external_ssh_ports, handle.docker_user,
                 handle.ssh_user)
 
-            locks.get_lock(lock_id).force_unlock()
-
     def _sync_workdir(self, handle: CloudVmRayResourceHandle,
                       workdir: Union[Path, Dict[str, Any]],
                       envs_and_secrets: Dict[str, str]) -> None:

sky/catalog/data_fetchers/fetch_shadeform.py

@@ -0,0 +1,142 @@
+"""A script that generates the Shadeform catalog.
+
+Usage:
+    python fetch_shadeform.py [-h] [--api-key API_KEY]
+                               [--api-key-path API_KEY_PATH]
+
+If neither --api-key nor --api-key-path are provided, this script will parse
+`~/.shadeform/api_key` to look for Shadeform API key.
+"""
+import argparse
+import csv
+import json
+import os
+from typing import Dict
+
+import requests
+
+ENDPOINT = 'https://api.shadeform.ai/v1/instances/types'
+DEFAULT_SHADEFORM_API_KEY_PATH = os.path.expanduser('~/.shadeform/api_key')
+
+
+def parse_gpu_info(gpu_type: str, num_gpus: int, ram_per_gpu: int) -> Dict:
+    """Parse GPU information for the catalog."""
+
+    manufacturer = 'NVIDIA'
+    if gpu_type == 'MI300X':
+        manufacturer = 'AMD'
+    elif gpu_type == 'GAUDI2':
+        manufacturer = 'Intel'
+
+    return {
+        'Gpus': [{
+            'Name': gpu_type,
+            'Manufacturer': manufacturer,
+            'Count': float(num_gpus),
+            'MemoryInfo': {
+                'SizeInMiB': ram_per_gpu
+            },
+            'TotalGpuMemoryInMiB': ram_per_gpu * num_gpus
+        }]
+    }
+
+
+def create_catalog(api_key: str, output_path: str) -> None:
+    """Create Shadeform catalog by fetching from API."""
+    headers = {'X-API-KEY': api_key}
+
+    params = {'available': 'true'}
+
+    response = requests.get(ENDPOINT,
+                            headers=headers,
+                            params=params,
+                            timeout=30)
+    response.raise_for_status()
+
+    data = response.json()
+    instance_types = data.get('instance_types', [])
+
+    with open(output_path, mode='w', encoding='utf-8') as f:
+        writer = csv.writer(f, delimiter=',', quotechar='"')
+        writer.writerow([
+            'InstanceType', 'AcceleratorName', 'AcceleratorCount', 'vCPUs',
+            'MemoryGiB', 'Price', 'Region', 'GpuInfo', 'SpotPrice'
+        ])
+
+        for instance in instance_types:
+            config = instance['configuration']
+
+            cloud = instance['cloud']
+            shade_instance_type = instance['shade_instance_type']
+            instance_type = f'{cloud}_{shade_instance_type.replace("_", "-")}'
+            gpu_type = config['gpu_type'].replace('_', '-')
+            gpu_count = float(config['num_gpus'])
+            vcpus = float(config['vcpus'])
+            memory_gb = int(config['memory_in_gb'])
+
+            # Append "B" to instance_type and gpu_type if they end with "G"
+            if instance_type.endswith('G'):
+                instance_type += 'B'
+            if gpu_type.endswith('G'):
+                gpu_type += 'B'
+
+            # Replace "Gx" with "GBx" (case sensitive)
+            if 'Gx' in instance_type:
+                instance_type = instance_type.replace('Gx', 'GBx')
+
+            # Price is in cents per hour, convert to dollars
+            price = float(instance['hourly_price']) / 100
+
+            # Create GPU info
+            gpuinfo = None
+            if gpu_count > 0:
+                gpuinfo_dict = parse_gpu_info(gpu_type, int(gpu_count),
+                                              int(config['vram_per_gpu_in_gb']))
+                gpuinfo = json.dumps(gpuinfo_dict).replace('"', '\'')
+
+            # Write entry for each available region
+            for availability in instance.get('availability', []):
+                if availability['available'] and gpu_count > 0:
+                    region = availability['region']
+                    writer.writerow([
+                        instance_type,
+                        gpu_type,
+                        gpu_count,
+                        vcpus,
+                        memory_gb,
+                        price,
+                        region,
+                        gpuinfo,
+                        ''  # No spot pricing info available
+                    ])
+
+
+def get_api_key(cmdline_args: argparse.Namespace) -> str:
+    """Get Shadeform API key from cmdline or default path."""
+    api_key = cmdline_args.api_key
+    if api_key is None:
+        if cmdline_args.api_key_path is not None:
+            with open(cmdline_args.api_key_path, mode='r',
+                      encoding='utf-8') as f:
+                api_key = f.read().strip()
+        else:
+            # Read from ~/.shadeform/api_key
+            with open(DEFAULT_SHADEFORM_API_KEY_PATH,
+                      mode='r',
+                      encoding='utf-8') as f:
+                api_key = f.read().strip()
+    assert api_key is not None, (
+        f'API key not found. Please provide via --api-key or place in '
+        f'{DEFAULT_SHADEFORM_API_KEY_PATH}')
+    return api_key
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--api-key', help='Shadeform API key.')
+    parser.add_argument('--api-key-path',
+                        help='path of file containing Shadeform API key.')
+    args = parser.parse_args()
+    os.makedirs('shadeform', exist_ok=True)
+    create_catalog(get_api_key(args), 'shadeform/vms.csv')
+    print('Shadeform catalog saved to shadeform/vms.csv')

sky/catalog/kubernetes_catalog.py

@@ -3,6 +3,7 @@
 Kubernetes does not require a catalog of instances, but we need an image catalog
 mapping SkyPilot image tags to corresponding container image tags.
 """
+import collections
 import re
 import typing
 from typing import Dict, List, Optional, Set, Tuple
@@ -167,12 +168,25 @@ def _list_accelerators(
     accelerators_qtys: Set[Tuple[str, int]] = set()
     keys = lf.get_label_keys()
     nodes = kubernetes_utils.get_kubernetes_nodes(context=context)
+
+    # Check if any nodes have accelerators before fetching pods
+    has_accelerator_nodes = False
+    for node in nodes:
+        for key in keys:
+            if key in node.metadata.labels:
+                has_accelerator_nodes = True
+                break
+        if has_accelerator_nodes:
+            break
+
+    # Only fetch pods if we have accelerator nodes and realtime is requested
     pods = None
-    if realtime:
-        # Get the pods to get the real-time GPU usage
+    allocated_qty_by_node: Dict[str, int] = collections.defaultdict(int)
+    if realtime and has_accelerator_nodes:
+        # Get the allocated GPU quantity by each node
         try:
-            pods = kubernetes_utils.get_all_pods_in_kubernetes_cluster(
-                context=context)
+            allocated_qty_by_node = (
+                kubernetes_utils.get_allocated_gpu_qty_by_node(context=context))
         except kubernetes.api_exception() as e:
             if e.status == 403:
                 logger.warning(
@@ -191,7 +205,6 @@ def _list_accelerators(
     for node in nodes:
         for key in keys:
             if key in node.metadata.labels:
-                allocated_qty = 0
                 accelerator_name = lf.get_accelerator_from_label_value(
                     node.metadata.labels.get(key))
 
@@ -251,26 +264,7 @@ def _list_accelerators(
                     total_accelerators_available[accelerator_name] = -1
                     continue
 
-                for pod in pods:
-                    # Get all the pods running on the node
-                    if (pod.spec.node_name == node.metadata.name and
-                            pod.status.phase in ['Running', 'Pending']):
-                        # Skip pods that should not count against GPU count
-                        if (kubernetes_utils.
-                                should_exclude_pod_from_gpu_allocation(pod)):
-                            logger.debug(
-                                f'Excluding pod '
-                                f'{pod.metadata.name} from GPU count '
-                                f'calculations on node {node.metadata.name}')
-                            continue
-                        # Iterate over all the containers in the pod and sum
-                        # the GPU requests
-                        for container in pod.spec.containers:
-                            if container.resources.requests:
-                                allocated_qty += (
-                                    kubernetes_utils.get_node_accelerator_count(
-                                        context, container.resources.requests))
-
+                allocated_qty = allocated_qty_by_node[node.metadata.name]
                 accelerators_available = accelerator_count - allocated_qty
                 # Initialize the total_accelerators_available to make sure the
                 # key exists in the dictionary.