skypilot-nightly 1.0.0.dev20251012__py3-none-any.whl → 1.0.0.dev20251013__py3-none-any.whl

sky/__init__.py

@@ -7,7 +7,7 @@ import urllib.request
 from sky.utils import directory_utils
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '7d5d1a2925fc7192af10061ca395d329364e7405'
+_SKYPILOT_COMMIT_SHA = 'c9063aba1971bd726c07f76164ad1127ac90424c'
 
 
 def _get_git_commit():
@@ -37,7 +37,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20251012'
+__version__ = '1.0.0.dev20251013'
 __root_dir__ = directory_utils.get_sky_dir()
 
 
@@ -150,6 +150,7 @@ Vsphere = clouds.Vsphere
 Fluidstack = clouds.Fluidstack
 Nebius = clouds.Nebius
 Hyperbolic = clouds.Hyperbolic
+Shadeform = clouds.Shadeform
 Seeweb = clouds.Seeweb
 
 __all__ = [
@@ -172,6 +173,7 @@ __all__ = [
     'Fluidstack',
     'Nebius',
     'Hyperbolic',
+    'Shadeform',
     'Seeweb',
     'Optimizer',
     'OptimizeTarget',

sky/adaptors/shadeform.py

@@ -0,0 +1,89 @@
+"""Shadeform cloud adaptor."""
+
+import functools
+import socket
+from typing import Any, Dict, List, Optional
+
+import requests
+
+from sky import sky_logging
+from sky.provision.shadeform import shadeform_utils
+from sky.utils import common_utils
+
+logger = sky_logging.init_logger(__name__)
+
+_shadeform_sdk = None
+
+
+def import_package(func):
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        global _shadeform_sdk
+        if _shadeform_sdk is None:
+            try:
+                import shadeform as _shadeform  # pylint: disable=import-outside-toplevel
+                _shadeform_sdk = _shadeform
+            except ImportError:
+                raise ImportError(
+                    'Failed to import dependencies for Shadeform. '
+                    'Try pip install "skypilot[shadeform]"') from None
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+@import_package
+def shadeform():
+    """Return the shadeform package."""
+    return _shadeform_sdk
+
+
+def list_ssh_keys() -> List[Dict[str, Any]]:
+    """List all SSH keys in Shadeform account."""
+    try:
+        response = shadeform_utils.get_ssh_keys()
+        return response.get('ssh_keys', [])
+    except (ValueError, KeyError, requests.exceptions.RequestException) as e:
+        logger.warning(f'Failed to list SSH keys from Shadeform: {e}')
+        return []
+
+
+def add_ssh_key_to_shadeform(public_key: str) -> Optional[str]:
+    """Add SSH key to Shadeform if it doesn't already exist.
+
+    Args:
+        public_key: The SSH public key string.
+
+    Returns:
+        The name of the key if added successfully, None otherwise.
+    """
+    try:
+        # Check if key already exists
+        existing_keys = list_ssh_keys()
+        key_exists = False
+        key_id = None
+        for key in existing_keys:
+            if key.get('public_key') == public_key:
+                key_exists = True
+                key_id = key.get('id')
+                break
+
+        if key_exists:
+            logger.info('SSH key already exists in Shadeform account')
+            return key_id
+
+        # Generate a unique key name
+        hostname = socket.gethostname()
+        key_name = f'skypilot-{hostname}-{common_utils.get_user_hash()[:8]}'
+
+        # Add the key
+        response = shadeform_utils.add_ssh_key(name=key_name,
+                                               public_key=public_key)
+        key_id = response['id']
+        logger.info(f'Added SSH key to Shadeform: {key_name, key_id}')
+        return key_id
+
+    except (ValueError, KeyError, requests.exceptions.RequestException) as e:
+        logger.warning(f'Failed to add SSH key to Shadeform: {e}')
+        return None

sky/authentication.py

@@ -39,6 +39,7 @@ from sky.adaptors import gcp
 from sky.adaptors import ibm
 from sky.adaptors import runpod
 from sky.adaptors import seeweb as seeweb_adaptor
+from sky.adaptors import shadeform as shadeform_adaptor
 from sky.adaptors import vast
 from sky.provision.fluidstack import fluidstack_utils
 from sky.provision.kubernetes import utils as kubernetes_utils
@@ -511,6 +512,48 @@ def setup_hyperbolic_authentication(config: Dict[str, Any]) -> Dict[str, Any]:
     return configure_ssh_info(config)
 
 
+def setup_shadeform_authentication(config: Dict[str, Any]) -> Dict[str, Any]:
+    """Sets up SSH authentication for Shadeform.
+    - Generates a new SSH key pair if one does not exist.
+    - Adds the public SSH key to the user's Shadeform account.
+
+    Note: This assumes there is a Shadeform Python SDK available.
+    If no official SDK exists, this function would need to use direct API calls.
+    """
+
+    _, public_key_path = get_or_generate_keys()
+    ssh_key_id = None
+
+    with open(public_key_path, 'r', encoding='utf-8') as f:
+        public_key = f.read().strip()
+
+    try:
+        # Add SSH key to Shadeform using our utility functions
+        ssh_key_id = shadeform_adaptor.add_ssh_key_to_shadeform(public_key)
+
+    except ImportError as e:
+        # If required dependencies are missing
+        logger.warning(
+            f'Failed to add Shadeform SSH key due to missing dependencies: '
+            f'{e}. Manually configure SSH keys in your Shadeform account.')
+
+    except Exception as e:
+        logger.warning(f'Failed to set up Shadeform authentication: {e}')
+        raise exceptions.CloudUserIdentityError(
+            'Failed to set up SSH authentication for Shadeform. '
+            f'Please ensure your Shadeform credentials are configured: {e}'
+        ) from e
+
+    if ssh_key_id is None:
+        raise Exception('Failed to add SSH key to Shadeform')
+
+    # Configure SSH info in the config
+    config['auth']['ssh_public_key'] = public_key_path
+    config['auth']['ssh_key_id'] = ssh_key_id
+
+    return configure_ssh_info(config)
+
+
 def setup_primeintellect_authentication(
         config: Dict[str, Any]) -> Dict[str, Any]:
     """Sets up SSH authentication for Prime Intellect.

sky/backends/backend_utils.py

@@ -1124,6 +1124,8 @@ def _add_auth_to_cluster_config(cloud: clouds.Cloud, tmp_yaml_path: str):
         config = auth.setup_fluidstack_authentication(config)
     elif isinstance(cloud, clouds.Hyperbolic):
         config = auth.setup_hyperbolic_authentication(config)
+    elif isinstance(cloud, clouds.Shadeform):
+        config = auth.setup_shadeform_authentication(config)
     elif isinstance(cloud, clouds.PrimeIntellect):
         config = auth.setup_primeintellect_authentication(config)
     elif isinstance(cloud, clouds.Seeweb):

sky/backends/cloud_vm_ray_backend.py

@@ -141,6 +141,7 @@ _NODES_LAUNCHING_PROGRESS_TIMEOUT = {
     clouds.OCI: 300,
     clouds.Paperspace: 600,
     clouds.Kubernetes: 300,
+    clouds.Shadeform: 300,
     clouds.Vsphere: 240,
 }
 
@@ -304,6 +305,7 @@ def _get_cluster_config_template(cloud):
         clouds.RunPod: 'runpod-ray.yml.j2',
         clouds.Kubernetes: 'kubernetes-ray.yml.j2',
         clouds.SSH: 'kubernetes-ray.yml.j2',
+        clouds.Shadeform: 'shadeform-ray.yml.j2',
         clouds.Vsphere: 'vsphere-ray.yml.j2',
         clouds.Vast: 'vast-ray.yml.j2',
         clouds.Fluidstack: 'fluidstack-ray.yml.j2',

sky/catalog/data_fetchers/fetch_shadeform.py

@@ -0,0 +1,142 @@
+"""A script that generates the Shadeform catalog.
+
+Usage:
+    python fetch_shadeform.py [-h] [--api-key API_KEY]
+                               [--api-key-path API_KEY_PATH]
+
+If neither --api-key nor --api-key-path are provided, this script will parse
+`~/.shadeform/api_key` to look for Shadeform API key.
+"""
+import argparse
+import csv
+import json
+import os
+from typing import Dict
+
+import requests
+
+ENDPOINT = 'https://api.shadeform.ai/v1/instances/types'
+DEFAULT_SHADEFORM_API_KEY_PATH = os.path.expanduser('~/.shadeform/api_key')
+
+
+def parse_gpu_info(gpu_type: str, num_gpus: int, ram_per_gpu: int) -> Dict:
+    """Parse GPU information for the catalog."""
+
+    manufacturer = 'NVIDIA'
+    if gpu_type == 'MI300X':
+        manufacturer = 'AMD'
+    elif gpu_type == 'GAUDI2':
+        manufacturer = 'Intel'
+
+    return {
+        'Gpus': [{
+            'Name': gpu_type,
+            'Manufacturer': manufacturer,
+            'Count': float(num_gpus),
+            'MemoryInfo': {
+                'SizeInMiB': ram_per_gpu
+            },
+            'TotalGpuMemoryInMiB': ram_per_gpu * num_gpus
+        }]
+    }
+
+
+def create_catalog(api_key: str, output_path: str) -> None:
+    """Create Shadeform catalog by fetching from API."""
+    headers = {'X-API-KEY': api_key}
+
+    params = {'available': 'true'}
+
+    response = requests.get(ENDPOINT,
+                            headers=headers,
+                            params=params,
+                            timeout=30)
+    response.raise_for_status()
+
+    data = response.json()
+    instance_types = data.get('instance_types', [])
+
+    with open(output_path, mode='w', encoding='utf-8') as f:
+        writer = csv.writer(f, delimiter=',', quotechar='"')
+        writer.writerow([
+            'InstanceType', 'AcceleratorName', 'AcceleratorCount', 'vCPUs',
+            'MemoryGiB', 'Price', 'Region', 'GpuInfo', 'SpotPrice'
+        ])
+
+        for instance in instance_types:
+            config = instance['configuration']
+
+            cloud = instance['cloud']
+            shade_instance_type = instance['shade_instance_type']
+            instance_type = f'{cloud}_{shade_instance_type.replace("_", "-")}'
+            gpu_type = config['gpu_type'].replace('_', '-')
+            gpu_count = float(config['num_gpus'])
+            vcpus = float(config['vcpus'])
+            memory_gb = int(config['memory_in_gb'])
+
+            # Append "B" to instance_type and gpu_type if they end with "G"
+            if instance_type.endswith('G'):
+                instance_type += 'B'
+            if gpu_type.endswith('G'):
+                gpu_type += 'B'
+
+            # Replace "Gx" with "GBx" (case sensitive)
+            if 'Gx' in instance_type:
+                instance_type = instance_type.replace('Gx', 'GBx')
+
+            # Price is in cents per hour, convert to dollars
+            price = float(instance['hourly_price']) / 100
+
+            # Create GPU info
+            gpuinfo = None
+            if gpu_count > 0:
+                gpuinfo_dict = parse_gpu_info(gpu_type, int(gpu_count),
+                                              int(config['vram_per_gpu_in_gb']))
+                gpuinfo = json.dumps(gpuinfo_dict).replace('"', '\'')
+
+            # Write entry for each available region
+            for availability in instance.get('availability', []):
+                if availability['available'] and gpu_count > 0:
+                    region = availability['region']
+                    writer.writerow([
+                        instance_type,
+                        gpu_type,
+                        gpu_count,
+                        vcpus,
+                        memory_gb,
+                        price,
+                        region,
+                        gpuinfo,
+                        ''  # No spot pricing info available
+                    ])
+
+
+def get_api_key(cmdline_args: argparse.Namespace) -> str:
+    """Get Shadeform API key from cmdline or default path."""
+    api_key = cmdline_args.api_key
+    if api_key is None:
+        if cmdline_args.api_key_path is not None:
+            with open(cmdline_args.api_key_path, mode='r',
+                      encoding='utf-8') as f:
+                api_key = f.read().strip()
+        else:
+            # Read from ~/.shadeform/api_key
+            with open(DEFAULT_SHADEFORM_API_KEY_PATH,
+                      mode='r',
+                      encoding='utf-8') as f:
+                api_key = f.read().strip()
+    assert api_key is not None, (
+        f'API key not found. Please provide via --api-key or place in '
+        f'{DEFAULT_SHADEFORM_API_KEY_PATH}')
+    return api_key
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--api-key', help='Shadeform API key.')
+    parser.add_argument('--api-key-path',
+                        help='path of file containing Shadeform API key.')
+    args = parser.parse_args()
+    os.makedirs('shadeform', exist_ok=True)
+    create_catalog(get_api_key(args), 'shadeform/vms.csv')
+    print('Shadeform catalog saved to shadeform/vms.csv')

sky/catalog/shadeform_catalog.py

@@ -0,0 +1,165 @@
+""" Shadeform | Catalog
+
+This module loads pricing and instance information from the Shadeform API
+and can be used to query instance types and pricing information for Shadeform.
+"""
+
+import typing
+from typing import Dict, List, Optional, Tuple, Union
+
+import pandas as pd
+
+from sky.catalog import common
+
+if typing.TYPE_CHECKING:
+    from sky.clouds import cloud
+
+# We'll use dynamic fetching, so no static CSV file to load
+_df = None
+
+
+def _get_df():
+    """Get the dataframe, fetching from API if needed."""
+    global _df
+    if _df is None:
+        # For now, we'll fall back to a minimal static catalog
+        # In a full implementation, this would call the Shadeform API
+        # to dynamically fetch the latest instance types and pricing
+        try:
+            df = common.read_catalog('shadeform/vms.csv')
+        except FileNotFoundError:
+            # If no static catalog exists, create an empty one
+            # This would be replaced with dynamic API fetching
+            _df = pd.DataFrame(columns=[
+                'InstanceType', 'AcceleratorName', 'AcceleratorCount', 'vCPUs',
+                'MemoryGiB', 'Price', 'Region', 'GpuInfo', 'SpotPrice'
+            ])
+        else:
+            df = df[df['InstanceType'].notna()]
+            if 'AcceleratorName' in df.columns:
+                df = df[df['AcceleratorName'].notna()]
+                df = df.assign(AcceleratorName=df['AcceleratorName'].astype(
+                    str).str.strip())
+            _df = df.reset_index(drop=True)
+    return _df
+
+
+def _is_not_found_error(err: ValueError) -> bool:
+    msg = str(err).lower()
+    return 'not found' in msg or 'not supported' in msg
+
+
+def _call_or_default(func, default):
+    try:
+        return func()
+    except ValueError as err:
+        if _is_not_found_error(err):
+            return default
+        raise
+
+
+def instance_type_exists(instance_type: str) -> bool:
+    """Check if an instance type exists."""
+    return common.instance_type_exists_impl(_get_df(), instance_type)
+
+
+def validate_region_zone(
+        region: Optional[str],
+        zone: Optional[str]) -> Tuple[Optional[str], Optional[str]]:
+    """Validate region and zone for Shadeform."""
+    return common.validate_region_zone_impl('shadeform', _get_df(), region,
+                                            zone)
+
+
+def get_hourly_cost(instance_type: str,
+                    use_spot: bool = False,
+                    region: Optional[str] = None,
+                    zone: Optional[str] = None) -> float:
+    """Returns the cost, or the cheapest cost among all zones for spot."""
+    # Shadeform doesn't support spot instances currently
+    if use_spot:
+        raise ValueError('Spot instances are not supported on Shadeform')
+
+    return common.get_hourly_cost_impl(_get_df(), instance_type, use_spot,
+                                       region, zone)
+
+
+def get_vcpus_mem_from_instance_type(
+        instance_type: str) -> Tuple[Optional[float], Optional[float]]:
+    """Get vCPUs and memory from instance type."""
+    return _call_or_default(
+        lambda: common.get_vcpus_mem_from_instance_type_impl(
+            _get_df(), instance_type), (None, None))
+
+
+def get_default_instance_type(cpus: Optional[str] = None,
+                              memory: Optional[str] = None,
+                              disk_tier: Optional[str] = None,
+                              region: Optional[str] = None,
+                              zone: Optional[str] = None) -> Optional[str]:
+    """Get default instance type based on requirements."""
+    del disk_tier  # Shadeform doesn't support custom disk tiers yet
+    return _call_or_default(
+        lambda: common.get_instance_type_for_cpus_mem_impl(
+            _get_df(), cpus, memory, region, zone), None)
+
+
+def get_accelerators_from_instance_type(
+        instance_type: str) -> Optional[Dict[str, Union[int, float]]]:
+    """Get accelerator information from instance type."""
+    return _call_or_default(
+        lambda: common.get_accelerators_from_instance_type_impl(
+            _get_df(), instance_type), None)
+
+
+def get_instance_type_for_accelerator(
+        acc_name: str,
+        acc_count: int,
+        cpus: Optional[str] = None,
+        memory: Optional[str] = None,
+        use_spot: bool = False,
+        region: Optional[str] = None,
+        zone: Optional[str] = None) -> Tuple[Optional[List[str]], List[str]]:
+    """Returns a list of instance types that have the given accelerator."""
+    if use_spot:
+        # Return empty lists since spot is not supported
+        return None, ['Spot instances are not supported on Shadeform']
+
+    return _call_or_default(
+        lambda: common.get_instance_type_for_accelerator_impl(
+            df=_get_df(),
+            acc_name=acc_name,
+            acc_count=acc_count,
+            cpus=cpus,
+            memory=memory,
+            use_spot=use_spot,
+            region=region,
+            zone=zone), (None, []))
+
+
+def get_region_zones_for_instance_type(instance_type: str,
+                                       use_spot: bool) -> List['cloud.Region']:
+    """Get regions and zones for an instance type."""
+    if use_spot:
+        return []  # No spot support
+
+    df = _get_df()
+    df_filtered = df[df['InstanceType'] == instance_type]
+    return _call_or_default(
+        lambda: common.get_region_zones(df_filtered, use_spot), [])
+
+
+def list_accelerators(
+        gpus_only: bool,
+        name_filter: Optional[str],
+        region_filter: Optional[str],
+        quantity_filter: Optional[int],
+        case_sensitive: bool = True,
+        all_regions: bool = False,
+        require_price: bool = True) -> Dict[str, List[common.InstanceTypeInfo]]:
+    """Returns all instance types in Shadeform offering GPUs."""
+    del require_price  # Unused.
+    return common.list_accelerators_impl('Shadeform', _get_df(), gpus_only,
+                                         name_filter, region_filter,
+                                         quantity_filter, case_sensitive,
+                                         all_regions)

sky/clouds/__init__.py

@@ -30,6 +30,7 @@ from sky.clouds.primeintellect import PrimeIntellect
 from sky.clouds.runpod import RunPod
 from sky.clouds.scp import SCP
 from sky.clouds.seeweb import Seeweb
+from sky.clouds.shadeform import Shadeform
 from sky.clouds.ssh import SSH
 from sky.clouds.vast import Vast
 from sky.clouds.vsphere import Vsphere
@@ -48,6 +49,7 @@ __all__ = [
     'PrimeIntellect',
     'SCP',
     'RunPod',
+    'Shadeform',
     'Vast',
     'OCI',
     'Vsphere',