skypilot-nightly 1.0.0.dev20250502__py3-none-any.whl → 1.0.0.dev20251203__py3-none-any.whl

sky/provision/shadeform/__init__.py

@@ -0,0 +1,11 @@
+"""Shadeform provisioner."""
+
+from sky.provision.shadeform.config import bootstrap_instances
+from sky.provision.shadeform.instance import cleanup_ports
+from sky.provision.shadeform.instance import get_cluster_info
+from sky.provision.shadeform.instance import open_ports
+from sky.provision.shadeform.instance import query_instances
+from sky.provision.shadeform.instance import run_instances
+from sky.provision.shadeform.instance import stop_instances
+from sky.provision.shadeform.instance import terminate_instances
+from sky.provision.shadeform.instance import wait_instances

sky/provision/shadeform/config.py

@@ -0,0 +1,12 @@
+"""Shadeform configuration bootstrapping."""
+
+from sky.provision import common
+
+
+def bootstrap_instances(
+        region: str, cluster_name: str,
+        config: common.ProvisionConfig) -> common.ProvisionConfig:
+    """Bootstraps instances for the given cluster."""
+    del region, cluster_name  # unused
+
+    return config

sky/provision/shadeform/instance.py

@@ -0,0 +1,351 @@
+"""Shadeform instance provisioning."""
+import time
+from typing import Any, Dict, List, Optional, Tuple
+
+import requests
+
+from sky import sky_logging
+from sky.provision import common
+from sky.provision.shadeform import shadeform_utils
+from sky.utils import status_lib
+
+POLL_INTERVAL = 10
+INSTANCE_READY_TIMEOUT = 3600
+
+logger = sky_logging.init_logger(__name__)
+
+# Status mapping from Shadeform to SkyPilot
+SHADEFORM_STATUS_MAP = {
+    'creating': status_lib.ClusterStatus.INIT,
+    'pending_provider': status_lib.ClusterStatus.INIT,
+    'pending': status_lib.ClusterStatus.INIT,
+    'active': status_lib.ClusterStatus.UP,
+    'deleted': status_lib.ClusterStatus.STOPPED,
+}
+
+
+def _get_cluster_instances(cluster_name_on_cloud: str) -> Dict[str, Any]:
+    """Get all instances belonging to a cluster."""
+    try:
+        response = shadeform_utils.get_instances()
+        instances = response.get('instances', [])
+
+        cluster_instances = {}
+        possible_names = [
+            f'{cluster_name_on_cloud}-head', f'{cluster_name_on_cloud}-worker'
+        ]
+
+        for instance in instances:
+            if instance.get('name') in possible_names:
+                cluster_instances[instance['id']] = instance
+
+        return cluster_instances
+    except (ValueError, KeyError, requests.exceptions.RequestException) as e:
+        logger.warning(f'Failed to get instances: {e}')
+        return {}
+
+
+def _get_head_instance_id(instances: Dict[str, Any]) -> Optional[str]:
+    """Get the head instance ID from a list of instances."""
+    for instance_id, instance in instances.items():
+        if instance.get('name', '').endswith('-head'):
+            return instance_id
+    return None
+
+
+def _wait_for_instances_ready(cluster_name_on_cloud: str,
+                              expected_count: int,
+                              timeout: int = INSTANCE_READY_TIMEOUT) -> bool:
+    """Wait for instances to be ready (active state with SSH access)."""
+    start_time = time.time()
+
+    while time.time() - start_time < timeout:
+        instances = _get_cluster_instances(cluster_name_on_cloud)
+        ready_count = 0
+
+        for instance in instances.values():
+            if (instance.get('status') == 'active' and
+                    instance.get('ip') is not None and
+                    instance.get('ssh_port') is not None):
+                ready_count += 1
+
+        logger.info(f'Waiting for instances to be ready: '
+                    f'({ready_count}/{expected_count})')
+
+        if ready_count >= expected_count:
+            return True
+
+        time.sleep(POLL_INTERVAL)
+
+    return False
+
+
+def run_instances(region: str, cluster_name: str, cluster_name_on_cloud: str,
+                  config: common.ProvisionConfig) -> common.ProvisionRecord:
+    """Run instances for the given cluster."""
+    del cluster_name  # unused - we use cluster_name_on_cloud
+    logger.info(f'Running instances for cluster {cluster_name_on_cloud} '
+                f'in region {region}')
+    logger.debug(f'DEBUG: region type={type(region)}, value={region!r}')
+    logger.debug(f'DEBUG: config node_config={config.node_config}')
+
+    # Check existing instances
+    existing_instances = _get_cluster_instances(cluster_name_on_cloud)
+    head_instance_id = _get_head_instance_id(existing_instances)
+
+    # Filter active instances
+    active_instances = {
+        iid: inst
+        for iid, inst in existing_instances.items()
+        if inst.get('status') == 'active'
+    }
+
+    current_count = len(active_instances)
+    target_count = config.count
+
+    logger.info(f'Current instances: {current_count}, target: {target_count}')
+
+    if current_count >= target_count:
+        if head_instance_id is None:
+            raise RuntimeError(
+                f'Cluster {cluster_name_on_cloud} has no head node')
+        logger.info(f'Cluster already has {current_count} instances, '
+                    f'no need to start more')
+        return common.ProvisionRecord(
+            provider_name='shadeform',
+            cluster_name=cluster_name_on_cloud,
+            region=region,
+            zone=None,  # Shadeform doesn't use separate zones
+            head_instance_id=head_instance_id,
+            resumed_instance_ids=[],
+            created_instance_ids=[])
+
+    # Create new instances
+    to_create = target_count - current_count
+    created_instance_ids = []
+
+    for _ in range(to_create):
+        node_type = 'head' if head_instance_id is None else 'worker'
+        instance_name = f'{cluster_name_on_cloud}-{node_type}'
+
+        # Extract configuration from node_config
+
+        # The node_config contains instance specs including InstanceType
+        # which follows the format: {cloud_provider}_{instance_type}
+        # (e.g., "massedcompute_A6000_basex2")
+        node_config = config.node_config
+        assert 'InstanceType' in node_config, \
+            'InstanceType must be present in node_config'
+
+        # Parse the instance type to extract cloud provider and instance specs
+        # Expected format: "{cloud}_{instance_type}" where cloud is provider
+        # (massedcompute, scaleway, lambda, etc.)
+        instance_type_full = node_config['InstanceType']
+        assert (isinstance(instance_type_full, str) and
+                '_' in instance_type_full), \
+            f'InstanceType must be in format cloud_instance_type, got: ' \
+            f'{instance_type_full}'
+
+        instance_type_split = instance_type_full.split('_')
+        assert len(instance_type_split) >= 2, \
+            f'InstanceType must contain at least one underscore, got: ' \
+            f'{instance_type_full}'
+
+        # Extract cloud provider (first part) and instance type (remaining)
+        # Example: "massedcompute_A6000-basex2" -> cloud="massedcompute",
+        # instance_type="A6000-basex2"
+        cloud = instance_type_split[0]
+        instance_type = '_'.join(instance_type_split[1:])
+
+        # Shadeform uses underscores instead of hyphens
+        instance_type = instance_type.replace('-', '_')
+
+        if instance_type.endswith('B'):
+            instance_type = instance_type[:-1]
+
+        # Replace "GBx" with "Gx" (case sensitive)
+        if 'GBx' in instance_type:
+            instance_type = instance_type.replace('GBx', 'Gx')
+
+        assert cloud, 'Cloud provider cannot be empty'
+        assert instance_type, 'Instance type cannot be empty'
+
+        # Get SSH key ID for authentication - this is optional and may be None
+        ssh_key_id = config.authentication_config.get('ssh_key_id')
+
+        create_config = {
+            'cloud': cloud,
+            'region': region,
+            'shade_instance_type': instance_type,
+            'name': instance_name,
+            'ssh_key_id': ssh_key_id
+        }
+
+        try:
+            logger.info(f'Creating {node_type} instance: {instance_name}')
+            response = shadeform_utils.create_instance(create_config)
+            instance_id = response['id']
+            created_instance_ids.append(instance_id)
+
+            if head_instance_id is None:
+                head_instance_id = instance_id
+
+            logger.info(f'Created instance {instance_id} ({node_type})')
+
+        except Exception as e:
+            logger.error(f'Failed to create instance: {e}')
+            # Clean up any created instances
+            for iid in created_instance_ids:
+                try:
+                    shadeform_utils.delete_instance(iid)
+                except requests.exceptions.RequestException as cleanup_e:
+                    logger.warning(
+                        f'Failed to cleanup instance {iid}: {cleanup_e}')
+            raise
+
+    # Wait for all instances to be ready
+    logger.info('Waiting for instances to become ready...')
+    if not _wait_for_instances_ready(cluster_name_on_cloud, target_count):
+        raise RuntimeError('Timed out waiting for instances to be ready')
+
+    assert head_instance_id is not None, 'head_instance_id should not be None'
+
+    return common.ProvisionRecord(provider_name='shadeform',
+                                  cluster_name=cluster_name_on_cloud,
+                                  region=region,
+                                  zone=region,
+                                  head_instance_id=head_instance_id,
+                                  resumed_instance_ids=[],
+                                  created_instance_ids=created_instance_ids)
+
+
+def wait_instances(region: str, cluster_name_on_cloud: str,
+                   state: Optional[status_lib.ClusterStatus]) -> None:
+    """Wait for instances to reach the specified state."""
+    del region, cluster_name_on_cloud, state  # unused
+    # For Shadeform, instances are ready when they reach 'active' status
+    # This is already handled in run_instances
+
+
+def stop_instances(cluster_name_on_cloud: str,
+                   provider_config: Optional[Dict[str, Any]] = None,
+                   worker_only: bool = False) -> None:
+    """Stop instances (not supported by Shadeform)."""
+    del cluster_name_on_cloud, provider_config, worker_only  # unused
+    raise NotImplementedError(
+        'Stopping instances is not supported by Shadeform')
+
+
+def terminate_instances(cluster_name_on_cloud: str,
+                        provider_config: Optional[Dict[str, Any]] = None,
+                        worker_only: bool = False) -> None:
+    """Terminate instances."""
+    del provider_config  # unused
+    logger.info(f'Terminating instances for cluster {cluster_name_on_cloud}')
+
+    instances = _get_cluster_instances(cluster_name_on_cloud)
+
+    if not instances:
+        logger.info(f'No instances found for cluster {cluster_name_on_cloud}')
+        return
+
+    instances_to_delete = instances
+    if worker_only:
+        # Only delete worker nodes, not head
+        instances_to_delete = {
+            iid: inst
+            for iid, inst in instances.items()
+            if not inst.get('name', '').endswith('-head')
+        }
+
+    for instance_id, instance in instances_to_delete.items():
+        try:
+            logger.info(
+                f'Terminating instance {instance_id} ({instance.get("name")})')
+            shadeform_utils.delete_instance(instance_id)
+        except requests.exceptions.RequestException as e:
+            logger.warning(f'Failed to terminate instance {instance_id}: {e}')
+
+
+def get_cluster_info(
+        region: str,
+        cluster_name_on_cloud: str,
+        provider_config: Optional[Dict[str, Any]] = None) -> common.ClusterInfo:
+    """Get cluster information."""
+    del region, provider_config  # unused
+    instances = _get_cluster_instances(cluster_name_on_cloud)
+
+    if not instances:
+        return common.ClusterInfo(instances={},
+                                  head_instance_id=None,
+                                  provider_name='shadeform')
+
+    head_instance_id = _get_head_instance_id(instances)
+
+    # Convert instance format for ClusterInfo
+    cluster_instances = {}
+    for instance_id, instance in instances.items():
+        instance_info = common.InstanceInfo(
+            instance_id=instance_id,
+            internal_ip=instance.get('ip', ''),
+            external_ip=instance.get('ip', ''),
+            ssh_port=instance.get('ssh_port', 22),
+            tags={},
+        )
+        # ClusterInfo expects Dict[InstanceId, List[InstanceInfo]]
+        cluster_instances[instance_id] = [instance_info]
+
+    ssh_user = 'shadeform'  # default
+    if head_instance_id is not None:
+        ssh_user = instances.get(head_instance_id,
+                                 {}).get('ssh_user', 'shadeform')
+
+    return common.ClusterInfo(instances=cluster_instances,
+                              head_instance_id=head_instance_id,
+                              provider_name='shadeform',
+                              ssh_user=ssh_user)
+
+
+def query_instances(
+    cluster_name: str,
+    cluster_name_on_cloud: str,
+    provider_config: Optional[Dict[str, Any]] = None,
+    non_terminated_only: bool = True,
+) -> Dict[str, Tuple[Optional['status_lib.ClusterStatus'], Optional[str]]]:
+    """Query the status of instances."""
+    del cluster_name, provider_config  # unused
+    instances = _get_cluster_instances(cluster_name_on_cloud)
+
+    if not instances:
+        return {}
+
+    status_map: Dict[str, Tuple[Optional['status_lib.ClusterStatus'],
+                                Optional[str]]] = {}
+    for instance_id, instance in instances.items():
+        shadeform_status = instance.get('status', 'unknown')
+        sky_status = SHADEFORM_STATUS_MAP.get(shadeform_status,
+                                              status_lib.ClusterStatus.INIT)
+
+        if (non_terminated_only and
+                sky_status == status_lib.ClusterStatus.STOPPED):
+            continue
+
+        status_map[instance_id] = (sky_status, None)
+
+    return status_map
+
+
+def open_ports(cluster_name_on_cloud: str,
+               ports: List[str],
+               provider_config: Optional[Dict[str, Any]] = None) -> None:
+    """Open ports (not supported by Shadeform)."""
+    del cluster_name_on_cloud, ports, provider_config  # unused
+    raise NotImplementedError()
+
+
+def cleanup_ports(cluster_name_on_cloud: str,
+                  ports: List[str],
+                  provider_config: Optional[Dict[str, Any]] = None) -> None:
+    """Cleanup ports (not supported by Shadeform)."""
+    del cluster_name_on_cloud, ports, provider_config  # unused
+    # Nothing to cleanup since we don't support dynamic port opening

sky/provision/shadeform/shadeform_utils.py

@@ -0,0 +1,83 @@
+"""Shadeform API utilities."""
+
+import os
+from typing import Any, Dict
+
+from sky.adaptors import common
+
+# Lazy import to avoid dependency on external packages
+requests = common.LazyImport('requests')
+
+# Shadeform API configuration
+SHADEFORM_API_BASE = 'https://api.shadeform.ai/v1'
+SHADEFORM_API_KEY_PATH = '~/.shadeform/api_key'
+
+
+def get_api_key() -> str:
+    """Get Shadeform API key from file."""
+    api_key_path = os.path.expanduser(SHADEFORM_API_KEY_PATH)
+    if not os.path.exists(api_key_path):
+        raise FileNotFoundError(
+            f'Shadeform API key not found at {api_key_path}. '
+            'Please save your API key to this file.')
+
+    with open(api_key_path, 'r', encoding='utf-8') as f:
+        api_key = f.read().strip()
+
+    if not api_key:
+        raise ValueError(f'Shadeform API key is empty in {api_key_path}')
+
+    return api_key
+
+
+def make_request(method: str, endpoint: str, **kwargs) -> Any:
+    """Make a request to the Shadeform API."""
+    url = f'{SHADEFORM_API_BASE}/{endpoint.lstrip("/")}'
+    headers = {
+        'X-API-KEY': get_api_key(),
+        'Content-Type': 'application/json',
+    }
+
+    response = requests.request(method, url, headers=headers, **kwargs)
+    response.raise_for_status()
+
+    # Some APIs (like delete) return empty responses with just 200 status
+    if response.text.strip():
+        return response.json()
+    else:
+        # Return empty dict for empty responses (e.g., delete operations)
+        return {}
+
+
+def get_instances() -> Dict[str, Any]:
+    """Get all instances."""
+    return make_request('GET', '/instances')
+
+
+def get_instance_info(instance_id: str) -> Dict[str, Any]:
+    """Get information about a specific instance."""
+    return make_request('GET', f'/instances/{instance_id}/info')
+
+
+def create_instance(config: Dict[str, Any]) -> Dict[str, Any]:
+    """Create a new instance."""
+    return make_request('POST', '/instances/create', json=config)
+
+
+def delete_instance(instance_id: str) -> Dict[str, Any]:
+    """Delete an instance.
+
+    Note: Shadeform delete API returns empty response with 200 status.
+    """
+    return make_request('POST', f'/instances/{instance_id}/delete')
+
+
+def get_ssh_keys() -> Dict[str, Any]:
+    """Get all SSH keys."""
+    return make_request('GET', '/sshkeys')
+
+
+def add_ssh_key(name: str, public_key: str) -> Dict[str, Any]:
+    """Add a new SSH key."""
+    config = {'name': name, 'public_key': public_key}
+    return make_request('POST', '/sshkeys/add', json=config)

sky/provision/ssh/__init__.py

@@ -0,0 +1,18 @@
+"""SSH provisioner for SkyPilot.
+
+This module implements the provisioner interface for SSH targets.
+It reuses most of the functionality from the Kubernetes provisioner,
+since the SSH implementation is based on Kubernetes under the hood.
+"""
+
+from sky.provision.kubernetes.config import bootstrap_instances
+from sky.provision.kubernetes.instance import get_cluster_info
+from sky.provision.kubernetes.instance import get_command_runners
+from sky.provision.kubernetes.instance import query_instances
+from sky.provision.kubernetes.instance import run_instances
+from sky.provision.kubernetes.instance import stop_instances
+from sky.provision.kubernetes.instance import terminate_instances
+from sky.provision.kubernetes.instance import wait_instances
+from sky.provision.kubernetes.network import cleanup_ports
+from sky.provision.kubernetes.network import open_ports
+from sky.provision.kubernetes.network import query_ports

sky/provision/vast/instance.py

@@ -1,6 +1,6 @@
 """Vast instance provisioning."""
 import time
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Tuple
 
 from sky import sky_logging
 from sky.provision import common
@@ -39,14 +39,15 @@ def _filter_instances(cluster_name_on_cloud: str,
 
 def _get_head_instance_id(instances: Dict[str, Any]) -> Optional[str]:
     for inst_id, inst in instances.items():
-        if inst['name'].endswith('-head'):
+        if inst.get('name') and inst['name'].endswith('-head'):
             return inst_id
     return None
 
 
-def run_instances(region: str, cluster_name_on_cloud: str,
+def run_instances(region: str, cluster_name: str, cluster_name_on_cloud: str,
                   config: common.ProvisionConfig) -> common.ProvisionRecord:
     """Runs instances for the given cluster."""
+    del cluster_name  # unused
     pending_status = ['CREATED', 'RESTARTING']
 
     created_instance_ids = []
@@ -97,7 +98,8 @@ def run_instances(region: str, cluster_name_on_cloud: str,
                     region=region,
                     disk_size=config.node_config['DiskSize'],
                     preemptible=config.node_config['Preemptible'],
-                    image_name=config.node_config['ImageId'])
+                    image_name=config.node_config['ImageId'],
+                    ports=config.ports_to_open_on_launch)
             except Exception as e:  # pylint: disable=broad-except
                 logger.warning(f'run_instances error: {e}')
                 raise
@@ -215,12 +217,14 @@ def open_ports(
 
 
 def query_instances(
+    cluster_name: str,
     cluster_name_on_cloud: str,
     provider_config: Optional[Dict[str, Any]] = None,
     non_terminated_only: bool = True,
-) -> Dict[str, Optional[status_lib.ClusterStatus]]:
+    retry_if_missing: bool = False,
+) -> Dict[str, Tuple[Optional['status_lib.ClusterStatus'], Optional[str]]]:
     """See sky/provision/__init__.py"""
-
+    del cluster_name, retry_if_missing  # unused
     assert provider_config is not None, (cluster_name_on_cloud, provider_config)
     instances = _filter_instances(cluster_name_on_cloud, None)
     # "running", "frozen", "stopped", "unknown", "loading"
@@ -230,12 +234,13 @@ def query_instances(
         'STOPPED': status_lib.ClusterStatus.STOPPED,
         'RUNNING': status_lib.ClusterStatus.UP,
     }
-    statuses: Dict[str, Optional[status_lib.ClusterStatus]] = {}
+    statuses: Dict[str, Tuple[Optional['status_lib.ClusterStatus'],
+                              Optional[str]]] = {}
     for inst_id, inst in instances.items():
         status = status_map[inst['status']]
         if non_terminated_only and status is None:
             continue
-        statuses[inst_id] = status
+        statuses[inst_id] = (status, None)
     return statuses
 
 

sky/provision/vast/utils.py

@@ -5,7 +5,7 @@
 # python sdk.
 #
 """Vast library wrapper for SkyPilot."""
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
 
 from sky import sky_logging
 from sky.adaptors import vast
@@ -34,7 +34,8 @@ def list_instances() -> Dict[str, Dict[str, Any]]:
 
 
 def launch(name: str, instance_type: str, region: str, disk_size: int,
-           image_name: str, preemptible: bool) -> str:
+           image_name: str, ports: Optional[List[int]],
+           preemptible: bool) -> str:
     """Launches an instance with the given parameters.
 
     Converts the instance_type to the Vast GPU name, finds the specs for the
@@ -58,6 +59,8 @@ def launch(name: str, instance_type: str, region: str, disk_size: int,
          The disk size {xx} GB is not exactly matched the requested
          size {yy} GB. It is possible to charge extra cost on disk.
 
+      *  `ports`: This is a feature flag to expose ports to the internet.
+
       *  `geolocation`: Geolocation on Vast can be as specific as the
          host chooses to be. They can say, for instance, "Yutakachō,
          Shinagawa District, Tokyo, JP." Such a specific geolocation
@@ -78,10 +81,8 @@ def launch(name: str, instance_type: str, region: str, disk_size: int,
 		 amount of memory.
 
       *  Vast instance types are an invention for skypilot. Refer to
-         service_catalog/vast_catalog.py for the current construction
-         of the type.
-
-    """
+         catalog/vast_catalog.py for the current construction
+         of the type."""
     cpu_ram = float(instance_type.split('-')[-1]) / 1024
     gpu_name = instance_type.split('-')[1].replace('_', ' ')
     num_gpus = int(instance_type.split('-')[0].replace('x', ''))
@@ -104,11 +105,13 @@ def launch(name: str, instance_type: str, region: str, disk_size: int,
 
     instance_touse = instance_list[0]
 
+    port_map = ' '.join([f'-p {p}:{p}' for p in ports]) if ports else ''
+
     launch_params = {
         'id': instance_touse['id'],
         'direct': True,
         'ssh': True,
-        'env': '-e __SOURCE=skypilot',
+        'env': f'-e __SOURCE=skypilot {port_map}',
         'onstart_cmd': ';'.join([
             'touch ~/.no_auto_tmux',
             f'echo "{vast.vast().api_key_access}" > ~/.vast_api_key',