skypilot-nightly 1.0.0.dev20251009__py3-none-any.whl → 1.0.0.dev20251107__py3-none-any.whl

sky/provision/shadeform/instance.py

@@ -0,0 +1,351 @@
+"""Shadeform instance provisioning."""
+import time
+from typing import Any, Dict, List, Optional, Tuple
+
+import requests
+
+from sky import sky_logging
+from sky.provision import common
+from sky.provision.shadeform import shadeform_utils
+from sky.utils import status_lib
+
+POLL_INTERVAL = 10
+INSTANCE_READY_TIMEOUT = 3600
+
+logger = sky_logging.init_logger(__name__)
+
+# Status mapping from Shadeform to SkyPilot
+SHADEFORM_STATUS_MAP = {
+    'creating': status_lib.ClusterStatus.INIT,
+    'pending_provider': status_lib.ClusterStatus.INIT,
+    'pending': status_lib.ClusterStatus.INIT,
+    'active': status_lib.ClusterStatus.UP,
+    'deleted': status_lib.ClusterStatus.STOPPED,
+}
+
+
+def _get_cluster_instances(cluster_name_on_cloud: str) -> Dict[str, Any]:
+    """Get all instances belonging to a cluster."""
+    try:
+        response = shadeform_utils.get_instances()
+        instances = response.get('instances', [])
+
+        cluster_instances = {}
+        possible_names = [
+            f'{cluster_name_on_cloud}-head', f'{cluster_name_on_cloud}-worker'
+        ]
+
+        for instance in instances:
+            if instance.get('name') in possible_names:
+                cluster_instances[instance['id']] = instance
+
+        return cluster_instances
+    except (ValueError, KeyError, requests.exceptions.RequestException) as e:
+        logger.warning(f'Failed to get instances: {e}')
+        return {}
+
+
+def _get_head_instance_id(instances: Dict[str, Any]) -> Optional[str]:
+    """Get the head instance ID from a list of instances."""
+    for instance_id, instance in instances.items():
+        if instance.get('name', '').endswith('-head'):
+            return instance_id
+    return None
+
+
+def _wait_for_instances_ready(cluster_name_on_cloud: str,
+                              expected_count: int,
+                              timeout: int = INSTANCE_READY_TIMEOUT) -> bool:
+    """Wait for instances to be ready (active state with SSH access)."""
+    start_time = time.time()
+
+    while time.time() - start_time < timeout:
+        instances = _get_cluster_instances(cluster_name_on_cloud)
+        ready_count = 0
+
+        for instance in instances.values():
+            if (instance.get('status') == 'active' and
+                    instance.get('ip') is not None and
+                    instance.get('ssh_port') is not None):
+                ready_count += 1
+
+        logger.info(f'Waiting for instances to be ready: '
+                    f'({ready_count}/{expected_count})')
+
+        if ready_count >= expected_count:
+            return True
+
+        time.sleep(POLL_INTERVAL)
+
+    return False
+
+
+def run_instances(region: str, cluster_name: str, cluster_name_on_cloud: str,
+                  config: common.ProvisionConfig) -> common.ProvisionRecord:
+    """Run instances for the given cluster."""
+    del cluster_name  # unused - we use cluster_name_on_cloud
+    logger.info(f'Running instances for cluster {cluster_name_on_cloud} '
+                f'in region {region}')
+    logger.debug(f'DEBUG: region type={type(region)}, value={region!r}')
+    logger.debug(f'DEBUG: config node_config={config.node_config}')
+
+    # Check existing instances
+    existing_instances = _get_cluster_instances(cluster_name_on_cloud)
+    head_instance_id = _get_head_instance_id(existing_instances)
+
+    # Filter active instances
+    active_instances = {
+        iid: inst
+        for iid, inst in existing_instances.items()
+        if inst.get('status') == 'active'
+    }
+
+    current_count = len(active_instances)
+    target_count = config.count
+
+    logger.info(f'Current instances: {current_count}, target: {target_count}')
+
+    if current_count >= target_count:
+        if head_instance_id is None:
+            raise RuntimeError(
+                f'Cluster {cluster_name_on_cloud} has no head node')
+        logger.info(f'Cluster already has {current_count} instances, '
+                    f'no need to start more')
+        return common.ProvisionRecord(
+            provider_name='shadeform',
+            cluster_name=cluster_name_on_cloud,
+            region=region,
+            zone=None,  # Shadeform doesn't use separate zones
+            head_instance_id=head_instance_id,
+            resumed_instance_ids=[],
+            created_instance_ids=[])
+
+    # Create new instances
+    to_create = target_count - current_count
+    created_instance_ids = []
+
+    for _ in range(to_create):
+        node_type = 'head' if head_instance_id is None else 'worker'
+        instance_name = f'{cluster_name_on_cloud}-{node_type}'
+
+        # Extract configuration from node_config
+
+        # The node_config contains instance specs including InstanceType
+        # which follows the format: {cloud_provider}_{instance_type}
+        # (e.g., "massedcompute_A6000_basex2")
+        node_config = config.node_config
+        assert 'InstanceType' in node_config, \
+            'InstanceType must be present in node_config'
+
+        # Parse the instance type to extract cloud provider and instance specs
+        # Expected format: "{cloud}_{instance_type}" where cloud is provider
+        # (massedcompute, scaleway, lambda, etc.)
+        instance_type_full = node_config['InstanceType']
+        assert (isinstance(instance_type_full, str) and
+                '_' in instance_type_full), \
+            f'InstanceType must be in format cloud_instance_type, got: ' \
+            f'{instance_type_full}'
+
+        instance_type_split = instance_type_full.split('_')
+        assert len(instance_type_split) >= 2, \
+            f'InstanceType must contain at least one underscore, got: ' \
+            f'{instance_type_full}'
+
+        # Extract cloud provider (first part) and instance type (remaining)
+        # Example: "massedcompute_A6000-basex2" -> cloud="massedcompute",
+        # instance_type="A6000-basex2"
+        cloud = instance_type_split[0]
+        instance_type = '_'.join(instance_type_split[1:])
+
+        # Shadeform uses underscores instead of hyphens
+        instance_type = instance_type.replace('-', '_')
+
+        if instance_type.endswith('B'):
+            instance_type = instance_type[:-1]
+
+        # Replace "GBx" with "Gx" (case sensitive)
+        if 'GBx' in instance_type:
+            instance_type = instance_type.replace('GBx', 'Gx')
+
+        assert cloud, 'Cloud provider cannot be empty'
+        assert instance_type, 'Instance type cannot be empty'
+
+        # Get SSH key ID for authentication - this is optional and may be None
+        ssh_key_id = config.authentication_config.get('ssh_key_id')
+
+        create_config = {
+            'cloud': cloud,
+            'region': region,
+            'shade_instance_type': instance_type,
+            'name': instance_name,
+            'ssh_key_id': ssh_key_id
+        }
+
+        try:
+            logger.info(f'Creating {node_type} instance: {instance_name}')
+            response = shadeform_utils.create_instance(create_config)
+            instance_id = response['id']
+            created_instance_ids.append(instance_id)
+
+            if head_instance_id is None:
+                head_instance_id = instance_id
+
+            logger.info(f'Created instance {instance_id} ({node_type})')
+
+        except Exception as e:
+            logger.error(f'Failed to create instance: {e}')
+            # Clean up any created instances
+            for iid in created_instance_ids:
+                try:
+                    shadeform_utils.delete_instance(iid)
+                except requests.exceptions.RequestException as cleanup_e:
+                    logger.warning(
+                        f'Failed to cleanup instance {iid}: {cleanup_e}')
+            raise
+
+    # Wait for all instances to be ready
+    logger.info('Waiting for instances to become ready...')
+    if not _wait_for_instances_ready(cluster_name_on_cloud, target_count):
+        raise RuntimeError('Timed out waiting for instances to be ready')
+
+    assert head_instance_id is not None, 'head_instance_id should not be None'
+
+    return common.ProvisionRecord(provider_name='shadeform',
+                                  cluster_name=cluster_name_on_cloud,
+                                  region=region,
+                                  zone=region,
+                                  head_instance_id=head_instance_id,
+                                  resumed_instance_ids=[],
+                                  created_instance_ids=created_instance_ids)
+
+
+def wait_instances(region: str, cluster_name_on_cloud: str,
+                   state: Optional[status_lib.ClusterStatus]) -> None:
+    """Wait for instances to reach the specified state."""
+    del region, cluster_name_on_cloud, state  # unused
+    # For Shadeform, instances are ready when they reach 'active' status
+    # This is already handled in run_instances
+
+
+def stop_instances(cluster_name_on_cloud: str,
+                   provider_config: Optional[Dict[str, Any]] = None,
+                   worker_only: bool = False) -> None:
+    """Stop instances (not supported by Shadeform)."""
+    del cluster_name_on_cloud, provider_config, worker_only  # unused
+    raise NotImplementedError(
+        'Stopping instances is not supported by Shadeform')
+
+
+def terminate_instances(cluster_name_on_cloud: str,
+                        provider_config: Optional[Dict[str, Any]] = None,
+                        worker_only: bool = False) -> None:
+    """Terminate instances."""
+    del provider_config  # unused
+    logger.info(f'Terminating instances for cluster {cluster_name_on_cloud}')
+
+    instances = _get_cluster_instances(cluster_name_on_cloud)
+
+    if not instances:
+        logger.info(f'No instances found for cluster {cluster_name_on_cloud}')
+        return
+
+    instances_to_delete = instances
+    if worker_only:
+        # Only delete worker nodes, not head
+        instances_to_delete = {
+            iid: inst
+            for iid, inst in instances.items()
+            if not inst.get('name', '').endswith('-head')
+        }
+
+    for instance_id, instance in instances_to_delete.items():
+        try:
+            logger.info(
+                f'Terminating instance {instance_id} ({instance.get("name")})')
+            shadeform_utils.delete_instance(instance_id)
+        except requests.exceptions.RequestException as e:
+            logger.warning(f'Failed to terminate instance {instance_id}: {e}')
+
+
+def get_cluster_info(
+        region: str,
+        cluster_name_on_cloud: str,
+        provider_config: Optional[Dict[str, Any]] = None) -> common.ClusterInfo:
+    """Get cluster information."""
+    del region, provider_config  # unused
+    instances = _get_cluster_instances(cluster_name_on_cloud)
+
+    if not instances:
+        return common.ClusterInfo(instances={},
+                                  head_instance_id=None,
+                                  provider_name='shadeform')
+
+    head_instance_id = _get_head_instance_id(instances)
+
+    # Convert instance format for ClusterInfo
+    cluster_instances = {}
+    for instance_id, instance in instances.items():
+        instance_info = common.InstanceInfo(
+            instance_id=instance_id,
+            internal_ip=instance.get('ip', ''),
+            external_ip=instance.get('ip', ''),
+            ssh_port=instance.get('ssh_port', 22),
+            tags={},
+        )
+        # ClusterInfo expects Dict[InstanceId, List[InstanceInfo]]
+        cluster_instances[instance_id] = [instance_info]
+
+    ssh_user = 'shadeform'  # default
+    if head_instance_id is not None:
+        ssh_user = instances.get(head_instance_id,
+                                 {}).get('ssh_user', 'shadeform')
+
+    return common.ClusterInfo(instances=cluster_instances,
+                              head_instance_id=head_instance_id,
+                              provider_name='shadeform',
+                              ssh_user=ssh_user)
+
+
+def query_instances(
+    cluster_name: str,
+    cluster_name_on_cloud: str,
+    provider_config: Optional[Dict[str, Any]] = None,
+    non_terminated_only: bool = True,
+) -> Dict[str, Tuple[Optional['status_lib.ClusterStatus'], Optional[str]]]:
+    """Query the status of instances."""
+    del cluster_name, provider_config  # unused
+    instances = _get_cluster_instances(cluster_name_on_cloud)
+
+    if not instances:
+        return {}
+
+    status_map: Dict[str, Tuple[Optional['status_lib.ClusterStatus'],
+                                Optional[str]]] = {}
+    for instance_id, instance in instances.items():
+        shadeform_status = instance.get('status', 'unknown')
+        sky_status = SHADEFORM_STATUS_MAP.get(shadeform_status,
+                                              status_lib.ClusterStatus.INIT)
+
+        if (non_terminated_only and
+                sky_status == status_lib.ClusterStatus.STOPPED):
+            continue
+
+        status_map[instance_id] = (sky_status, None)
+
+    return status_map
+
+
+def open_ports(cluster_name_on_cloud: str,
+               ports: List[str],
+               provider_config: Optional[Dict[str, Any]] = None) -> None:
+    """Open ports (not supported by Shadeform)."""
+    del cluster_name_on_cloud, ports, provider_config  # unused
+    raise NotImplementedError()
+
+
+def cleanup_ports(cluster_name_on_cloud: str,
+                  ports: List[str],
+                  provider_config: Optional[Dict[str, Any]] = None) -> None:
+    """Cleanup ports (not supported by Shadeform)."""
+    del cluster_name_on_cloud, ports, provider_config  # unused
+    # Nothing to cleanup since we don't support dynamic port opening

sky/provision/shadeform/shadeform_utils.py

@@ -0,0 +1,83 @@
+"""Shadeform API utilities."""
+
+import os
+from typing import Any, Dict
+
+from sky.adaptors import common
+
+# Lazy import to avoid dependency on external packages
+requests = common.LazyImport('requests')
+
+# Shadeform API configuration
+SHADEFORM_API_BASE = 'https://api.shadeform.ai/v1'
+SHADEFORM_API_KEY_PATH = '~/.shadeform/api_key'
+
+
+def get_api_key() -> str:
+    """Get Shadeform API key from file."""
+    api_key_path = os.path.expanduser(SHADEFORM_API_KEY_PATH)
+    if not os.path.exists(api_key_path):
+        raise FileNotFoundError(
+            f'Shadeform API key not found at {api_key_path}. '
+            'Please save your API key to this file.')
+
+    with open(api_key_path, 'r', encoding='utf-8') as f:
+        api_key = f.read().strip()
+
+    if not api_key:
+        raise ValueError(f'Shadeform API key is empty in {api_key_path}')
+
+    return api_key
+
+
+def make_request(method: str, endpoint: str, **kwargs) -> Any:
+    """Make a request to the Shadeform API."""
+    url = f'{SHADEFORM_API_BASE}/{endpoint.lstrip("/")}'
+    headers = {
+        'X-API-KEY': get_api_key(),
+        'Content-Type': 'application/json',
+    }
+
+    response = requests.request(method, url, headers=headers, **kwargs)
+    response.raise_for_status()
+
+    # Some APIs (like delete) return empty responses with just 200 status
+    if response.text.strip():
+        return response.json()
+    else:
+        # Return empty dict for empty responses (e.g., delete operations)
+        return {}
+
+
+def get_instances() -> Dict[str, Any]:
+    """Get all instances."""
+    return make_request('GET', '/instances')
+
+
+def get_instance_info(instance_id: str) -> Dict[str, Any]:
+    """Get information about a specific instance."""
+    return make_request('GET', f'/instances/{instance_id}/info')
+
+
+def create_instance(config: Dict[str, Any]) -> Dict[str, Any]:
+    """Create a new instance."""
+    return make_request('POST', '/instances/create', json=config)
+
+
+def delete_instance(instance_id: str) -> Dict[str, Any]:
+    """Delete an instance.
+
+    Note: Shadeform delete API returns empty response with 200 status.
+    """
+    return make_request('POST', f'/instances/{instance_id}/delete')
+
+
+def get_ssh_keys() -> Dict[str, Any]:
+    """Get all SSH keys."""
+    return make_request('GET', '/sshkeys')
+
+
+def add_ssh_key(name: str, public_key: str) -> Dict[str, Any]:
+    """Add a new SSH key."""
+    config = {'name': name, 'public_key': public_key}
+    return make_request('POST', '/sshkeys/add', json=config)

sky/provision/vast/instance.py

@@ -221,9 +221,10 @@ def query_instances(
     cluster_name_on_cloud: str,
     provider_config: Optional[Dict[str, Any]] = None,
     non_terminated_only: bool = True,
+    retry_if_missing: bool = False,
 ) -> Dict[str, Tuple[Optional['status_lib.ClusterStatus'], Optional[str]]]:
     """See sky/provision/__init__.py"""
-    del cluster_name  # unused
+    del cluster_name, retry_if_missing  # unused
     assert provider_config is not None, (cluster_name_on_cloud, provider_config)
     instances = _filter_instances(cluster_name_on_cloud, None)
     # "running", "frozen", "stopped", "unknown", "loading"

sky/provision/vsphere/instance.py

@@ -398,9 +398,10 @@ def query_instances(
     cluster_name_on_cloud: str,
     provider_config: Optional[Dict[str, Any]] = None,
     non_terminated_only: bool = True,
+    retry_if_missing: bool = False,
 ) -> Dict[str, Tuple[Optional['status_lib.ClusterStatus'], Optional[str]]]:
     """See sky/provision/__init__.py"""
-    del cluster_name  # unused
+    del cluster_name, retry_if_missing  # unused
     logger.info('New provision of Vsphere: query_instances().')
     assert provider_config is not None, cluster_name_on_cloud
     region = provider_config['region']

sky/resources.py

@@ -1104,7 +1104,7 @@ class Resources:
         regions = self.cloud.regions_with_offering(self._instance_type,
                                                    self.accelerators,
                                                    self._use_spot, self._region,
-                                                   self._zone)
+                                                   self._zone, self)
         if self._image_id is not None and None not in self._image_id:
             regions = [r for r in regions if r.name in self._image_id]
 

sky/schemas/api/responses.py

@@ -77,8 +77,13 @@ class APIHealthResponse(ResponseBaseModel):
     version: str = ''
     version_on_disk: str = ''
     commit: str = ''
+    # Whether basic auth on api server is enabled
     basic_auth_enabled: bool = False
     user: Optional[models.User] = None
+    # Whether service account token is enabled
+    service_account_token_enabled: bool = False
+    # Whether basic auth on ingress is enabled
+    ingress_basic_auth_enabled: bool = False
 
 
 class StatusResponse(ResponseBaseModel):
@@ -90,7 +95,7 @@ class StatusResponse(ResponseBaseModel):
     # This is an internally facing field anyway, so it's less
     # of a problem that it's not typed.
     handle: Optional[Any] = None
-    last_use: str
+    last_use: Optional[str] = None
     status: status_lib.ClusterStatus
     autostop: int
     to_down: bool
@@ -98,11 +103,8 @@ class StatusResponse(ResponseBaseModel):
     # metadata is a JSON, so we use Any here.
     metadata: Optional[Dict[str, Any]] = None
     cluster_hash: str
-    # pydantic cannot generate the pydantic-core schema for
-    # storage_mounts_metadata, so we use Any here.
-    storage_mounts_metadata: Optional[Dict[str, Any]] = None
     cluster_ever_up: bool
-    status_updated_at: int
+    status_updated_at: Optional[int] = None
     user_hash: str
     user_name: str
     config_hash: Optional[str] = None
@@ -160,6 +162,8 @@ class StorageRecord(ResponseBaseModel):
 # and therefore can be non-optional.
 class ManagedJobRecord(ResponseBaseModel):
     """A single managed job record."""
+    # The job_id in the spot table
+    task_job_id: Optional[int] = pydantic.Field(None, alias='_job_id')
     job_id: Optional[int] = None
     task_id: Optional[int] = None
     job_name: Optional[str] = None

sky/schemas/db/skypilot_config/001_initial_schema.py

@@ -0,0 +1,30 @@
+"""Initial schema for sky config database
+
+Revision ID: 001
+Revises:
+Create Date: 2025-10-21
+
+"""
+# pylint: disable=invalid-name
+from alembic import op
+
+from sky.skypilot_config import Base
+from sky.utils.db import db_utils
+
+# revision identifiers, used by Alembic.
+revision = '001'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    """Create initial schema for config_yaml table"""
+    with op.get_context().autocommit_block():
+        # Create all tables with their current schema
+        db_utils.add_all_tables_to_db_sqlalchemy(Base.metadata, op.get_bind())
+
+
+def downgrade():
+    """Drop all tables"""
+    Base.metadata.drop_all(bind=op.get_bind())

sky/schemas/db/spot_jobs/004_job_file_contents.py

@@ -0,0 +1,42 @@
+"""Add columns for stored DAG/env file contents.
+
+Revision ID: 004
+Revises: 003
+Create Date: 2025-10-27
+
+"""
+# pylint: disable=invalid-name
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+from sky.utils.db import db_utils
+
+# revision identifiers, used by Alembic.
+revision: str = '004'
+down_revision: Union[str, Sequence[str], None] = '003'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade():
+    """Add columns to persist job file contents in the database."""
+    with op.get_context().autocommit_block():
+        db_utils.add_column_to_table_alembic('job_info',
+                                             'dag_yaml_content',
+                                             sa.Text(),
+                                             server_default=None)
+        db_utils.add_column_to_table_alembic('job_info',
+                                             'original_user_yaml_content',
+                                             sa.Text(),
+                                             server_default=None)
+        db_utils.add_column_to_table_alembic('job_info',
+                                             'env_file_content',
+                                             sa.Text(),
+                                             server_default=None)
+
+
+def downgrade():
+    """No downgrade logic."""
+    pass

sky/schemas/db/spot_jobs/005_logs_gc.py

@@ -0,0 +1,38 @@
+"""Adding columns for the GC time of task logs and controller logs.
+
+Revision ID: 005
+Revises: 004
+Create Date: 2025-10-20
+
+"""
+# pylint: disable=invalid-name
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+from sky.utils.db import db_utils
+
+# revision identifiers, used by Alembic.
+revision: str = '005'
+down_revision: Union[str, Sequence[str], None] = '004'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade():
+    """Add columns for logs gc."""
+    with op.get_context().autocommit_block():
+        db_utils.add_column_to_table_alembic('job_info',
+                                             'controller_logs_cleaned_at',
+                                             sa.Float(),
+                                             server_default=None)
+        db_utils.add_column_to_table_alembic('spot',
+                                             'logs_cleaned_at',
+                                             sa.Float(),
+                                             server_default=None)
+
+
+def downgrade():
+    """Remove columns for logs gc."""
+    pass