skypilot-nightly 1.0.0.dev20251009__py3-none-any.whl → 1.0.0.dev20251107__py3-none-any.whl

sky/__init__.py

@@ -7,7 +7,7 @@ import urllib.request
 from sky.utils import directory_utils
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '02a8e33361b9deca8588cc30221de41bbd5e5cf8'
+_SKYPILOT_COMMIT_SHA = 'd7530d48bc1a331b5644bd6fbbc51eaebf1432f3'
 
 
 def _get_git_commit():
@@ -37,7 +37,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20251009'
+__version__ = '1.0.0.dev20251107'
 __root_dir__ = directory_utils.get_sky_dir()
 
 
@@ -122,6 +122,7 @@ from sky.data import StoreType
 from sky.jobs import ManagedJobStatus
 from sky.optimizer import Optimizer
 from sky.resources import Resources
+from sky.server.requests.request_names import AdminPolicyRequestName
 from sky.skylet.job_lib import JobStatus
 from sky.task import Task
 from sky.utils.common import OptimizeTarget
@@ -150,6 +151,7 @@ Vsphere = clouds.Vsphere
 Fluidstack = clouds.Fluidstack
 Nebius = clouds.Nebius
 Hyperbolic = clouds.Hyperbolic
+Shadeform = clouds.Shadeform
 Seeweb = clouds.Seeweb
 
 __all__ = [
@@ -172,6 +174,7 @@ __all__ = [
     'Fluidstack',
     'Nebius',
     'Hyperbolic',
+    'Shadeform',
     'Seeweb',
     'Optimizer',
     'OptimizeTarget',
@@ -226,6 +229,7 @@ __all__ = [
     'MutatedUserRequest',
     'AdminPolicy',
     'Config',
+    'AdminPolicyRequestName',
     # Registry
     'CLOUD_REGISTRY',
     'JOBS_RECOVERY_STRATEGY_REGISTRY',

sky/adaptors/aws.py

@@ -34,6 +34,7 @@ import time
 import typing
 from typing import Callable, Literal, Optional, TypeVar
 
+from sky import skypilot_config
 from sky.adaptors import common
 from sky.utils import annotations
 from sky.utils import common_utils
@@ -119,12 +120,27 @@ def _create_aws_object(creation_fn_or_cls: Callable[[], T],
                         f'{common_utils.format_exception(e)}.')
 
 
+def get_workspace_profile() -> Optional[str]:
+    """Get AWS profile name from workspace config."""
+    return skypilot_config.get_workspace_cloud('aws').get('profile', None)
+
+
 # The LRU cache needs to be thread-local to avoid multiple threads sharing the
 # same session object, which is not guaranteed to be thread-safe.
 @_thread_local_lru_cache()
-def session(check_credentials: bool = True):
-    """Create an AWS session."""
-    s = _create_aws_object(boto3.session.Session, 'session')
+def session(check_credentials: bool = True, profile: Optional[str] = None):
+    """Create an AWS session.
+
+    Args:
+        check_credentials: Whether to check if credentials are available.
+        profile: AWS profile name to use. If None, uses default credentials.
+    """
+    if profile is not None:
+        logger.debug(f'Using AWS profile \'{profile}\'.')
+        s = _create_aws_object(
+            lambda: boto3.session.Session(profile_name=profile), 'session')
+    else:
+        s = _create_aws_object(boto3.session.Session, 'session')
     if check_credentials and s.get_credentials() is None:
         # s.get_credentials() can be None if there are actually no credentials,
         # or if we fail to get credentials from IMDS (e.g. due to throttling).
@@ -180,13 +196,14 @@ def resource(service_name: str, **kwargs):
         kwargs['config'] = config
 
     check_credentials = kwargs.pop('check_credentials', True)
+    profile = get_workspace_profile()
 
     # Need to use the client retrieved from the per-thread session to avoid
     # thread-safety issues (Directly creating the client with boto3.resource()
     # is not thread-safe). Reference: https://stackoverflow.com/a/59635814
     return _create_aws_object(
-        lambda: session(check_credentials=check_credentials).resource(
-            service_name, **kwargs), 'resource')
+        lambda: session(check_credentials=check_credentials, profile=profile).
+        resource(service_name, **kwargs), 'resource')
 
 
 # New typing overloads can be added as needed.
@@ -221,14 +238,15 @@ def client(service_name: str, **kwargs):
     _assert_kwargs_builtin_type(kwargs)
 
     check_credentials = kwargs.pop('check_credentials', True)
+    profile = get_workspace_profile()
 
     # Need to use the client retrieved from the per-thread session to avoid
     # thread-safety issues (Directly creating the client with boto3.client() is
     # not thread-safe). Reference: https://stackoverflow.com/a/59635814
 
     return _create_aws_object(
-        lambda: session(check_credentials=check_credentials).client(
-            service_name, **kwargs), 'client')
+        lambda: session(check_credentials=check_credentials, profile=profile).
+        client(service_name, **kwargs), 'client')
 
 
 @common.load_lazy_modules(modules=_LAZY_MODULES)

sky/adaptors/coreweave.py

@@ -0,0 +1,278 @@
+"""CoreWeave cloud adaptor."""
+
+import configparser
+import contextlib
+import os
+import threading
+from typing import Dict, Optional, Tuple
+
+from sky import exceptions
+from sky import sky_logging
+from sky.adaptors import common
+from sky.clouds import cloud
+from sky.utils import annotations
+from sky.utils import ux_utils
+
+logger = sky_logging.init_logger(__name__)
+
+COREWEAVE_PROFILE_NAME = 'cw'
+COREWEAVE_CREDENTIALS_PATH = '~/.coreweave/cw.credentials'
+COREWEAVE_CONFIG_PATH = '~/.coreweave/cw.config'
+NAME = 'CoreWeave'
+DEFAULT_REGION = 'US-EAST-01A'
+_DEFAULT_ENDPOINT = 'https://cwobject.com'
+_INDENT_PREFIX = '    '
+
+_IMPORT_ERROR_MESSAGE = ('Failed to import dependencies for CoreWeave.'
+                         'Try pip install "skypilot[coreweave]"')
+
+boto3 = common.LazyImport('boto3', import_error_message=_IMPORT_ERROR_MESSAGE)
+botocore = common.LazyImport('botocore',
+                             import_error_message=_IMPORT_ERROR_MESSAGE)
+
+_LAZY_MODULES = (boto3, botocore)
+_session_creation_lock = threading.RLock()
+
+
+@contextlib.contextmanager
+def _load_cw_credentials_env():
+    """Context manager to temporarily change the AWS credentials file path."""
+    prev_credentials_path = os.environ.get('AWS_SHARED_CREDENTIALS_FILE')
+    prev_config_path = os.environ.get('AWS_CONFIG_FILE')
+    os.environ['AWS_SHARED_CREDENTIALS_FILE'] = COREWEAVE_CREDENTIALS_PATH
+    os.environ['AWS_CONFIG_FILE'] = COREWEAVE_CONFIG_PATH
+    try:
+        yield
+    finally:
+        if prev_credentials_path is None:
+            del os.environ['AWS_SHARED_CREDENTIALS_FILE']
+        else:
+            os.environ['AWS_SHARED_CREDENTIALS_FILE'] = prev_credentials_path
+        if prev_config_path is None:
+            del os.environ['AWS_CONFIG_FILE']
+        else:
+            os.environ['AWS_CONFIG_FILE'] = prev_config_path
+
+
+def get_coreweave_credentials(boto3_session):
+    """Gets the CoreWeave credentials from the boto3 session object.
+
+    Args:
+        boto3_session: The boto3 session object.
+    Returns:
+        botocore.credentials.ReadOnlyCredentials object with the CoreWeave
+        credentials.
+    """
+    with _load_cw_credentials_env():
+        coreweave_credentials = boto3_session.get_credentials()
+        if coreweave_credentials is None:
+            with ux_utils.print_exception_no_traceback():
+                raise ValueError('CoreWeave credentials not found. Run '
+                                 '`sky check` to verify credentials are '
+                                 'correctly set up.')
+        return coreweave_credentials.get_frozen_credentials()
+
+
+@annotations.lru_cache(scope='global')
+def session():
+    """Create an AWS session for CoreWeave."""
+    # Creating the session object is not thread-safe for boto3,
+    # so we add a reentrant lock to synchronize the session creation.
+    # Reference: https://github.com/boto/boto3/issues/1592
+    # However, the session object itself is thread-safe, so we are
+    # able to use lru_cache() to cache the session object.
+    with _session_creation_lock:
+        with _load_cw_credentials_env():
+            session_ = boto3.session.Session(
+                profile_name=COREWEAVE_PROFILE_NAME)
+        return session_
+
+
+@annotations.lru_cache(scope='global')
+def resource(resource_name: str, **kwargs):
+    """Create a CoreWeave resource.
+
+    Args:
+        resource_name: CoreWeave resource name (e.g., 's3').
+        kwargs: Other options.
+    """
+    # Need to use the resource retrieved from the per-thread session
+    # to avoid thread-safety issues (Directly creating the client
+    # with boto3.resource() is not thread-safe).
+    # Reference: https://stackoverflow.com/a/59635814
+
+    session_ = session()
+    coreweave_credentials = get_coreweave_credentials(session_)
+    endpoint = get_endpoint()
+
+    return session_.resource(
+        resource_name,
+        endpoint_url=endpoint,
+        aws_access_key_id=coreweave_credentials.access_key,
+        aws_secret_access_key=coreweave_credentials.secret_key,
+        region_name='auto',
+        config=botocore.config.Config(s3={'addressing_style': 'virtual'}),
+        **kwargs)
+
+
+@annotations.lru_cache(scope='global')
+def client(service_name: str):
+    """Create CoreWeave client of a certain service.
+
+    Args:
+        service_name: CoreWeave service name (e.g., 's3').
+    """
+    # Need to use the client retrieved from the per-thread session
+    # to avoid thread-safety issues (Directly creating the client
+    # with boto3.client() is not thread-safe).
+    # Reference: https://stackoverflow.com/a/59635814
+
+    session_ = session()
+    coreweave_credentials = get_coreweave_credentials(session_)
+    endpoint = get_endpoint()
+
+    return session_.client(
+        service_name,
+        endpoint_url=endpoint,
+        aws_access_key_id=coreweave_credentials.access_key,
+        aws_secret_access_key=coreweave_credentials.secret_key,
+        region_name='auto',
+        config=botocore.config.Config(s3={'addressing_style': 'virtual'}),
+    )
+
+
+@common.load_lazy_modules(_LAZY_MODULES)
+def botocore_exceptions():
+    """AWS botocore exception."""
+    # pylint: disable=import-outside-toplevel
+    from botocore import exceptions as boto_exceptions
+    return boto_exceptions
+
+
+def get_endpoint():
+    """Parse the COREWEAVE_CONFIG_PATH to get the endpoint_url.
+
+    The config file is an AWS-style config file with format:
+        [profile cw]
+        endpoint_url = https://cwobject.com
+        s3 =
+            addressing_style = virtual
+
+    Returns:
+        str: The endpoint URL from the config file, or the default endpoint
+             if the file doesn't exist or doesn't contain the endpoint_url.
+    """
+    config_path = os.path.expanduser(COREWEAVE_CONFIG_PATH)
+    if not os.path.isfile(config_path):
+        return _DEFAULT_ENDPOINT
+
+    try:
+        config = configparser.ConfigParser()
+        config.read(config_path)
+
+        # Try to get endpoint_url from [profile cw] section
+        profile_section = f'profile {COREWEAVE_PROFILE_NAME}'
+        if config.has_section(profile_section):
+            if config.has_option(profile_section, 'endpoint_url'):
+                endpoint = config.get(profile_section, 'endpoint_url')
+                return endpoint.strip()
+    except (configparser.Error, OSError) as e:
+        logger.warning(f'Failed to parse CoreWeave config file: {e}. '
+                       f'Using default endpoint: {_DEFAULT_ENDPOINT}')
+
+    return _DEFAULT_ENDPOINT
+
+
+def check_credentials(
+        cloud_capability: cloud.CloudCapability) -> Tuple[bool, Optional[str]]:
+    if cloud_capability == cloud.CloudCapability.STORAGE:
+        return check_storage_credentials()
+    else:
+        raise exceptions.NotSupportedError(
+            f'{NAME} does not support {cloud_capability}.')
+
+
+def check_storage_credentials() -> Tuple[bool, Optional[str]]:
+    """Checks if the user has access credentials to CoreWeave Object Storage.
+
+    Returns:
+        A tuple of a boolean value and a hint message where the bool
+        is True when both credentials needed for CoreWeave storage is set.
+        It is False when either of those are not set, which would hint with a
+        string on unset credential.
+    """
+    hints = None
+    profile_in_cred = coreweave_profile_in_cred()
+    profile_in_config = coreweave_profile_in_config()
+
+    if not profile_in_cred:
+        hints = (f'[{COREWEAVE_PROFILE_NAME}] profile is not set in '
+                 f'{COREWEAVE_CREDENTIALS_PATH}.')
+    if not profile_in_config:
+        if hints:
+            hints += ' Additionally, '
+        else:
+            hints = ''
+        hints += (f'[{COREWEAVE_PROFILE_NAME}] profile is not set in '
+                  f'{COREWEAVE_CONFIG_PATH}.')
+
+    if hints:
+        hints += ' Run the following commands:'
+        if not profile_in_cred:
+            hints += f'\n{_INDENT_PREFIX}  $ pip install boto3'
+            hints += (f'\n{_INDENT_PREFIX}  $ AWS_SHARED_CREDENTIALS_FILE='
+                      f'{COREWEAVE_CREDENTIALS_PATH} aws configure --profile '
+                      f'{COREWEAVE_PROFILE_NAME}')
+        if not profile_in_config:
+            hints += (f'\n{_INDENT_PREFIX}  $ AWS_CONFIG_FILE='
+                      f'{COREWEAVE_CONFIG_PATH} aws configure set endpoint_url'
+                      f' <ENDPOINT_URL> --profile '
+                      f'{COREWEAVE_PROFILE_NAME}')
+            hints += (f'\n{_INDENT_PREFIX}  $ AWS_CONFIG_FILE='
+                      f'{COREWEAVE_CONFIG_PATH} aws configure set '
+                      f's3.addressing_style virtual --profile '
+                      f'{COREWEAVE_PROFILE_NAME}')
+        hints += f'\n{_INDENT_PREFIX}For more info: '
+        hints += 'https://docs.coreweave.com/docs/products/storage/object-storage/get-started-caios'  # pylint: disable=line-too-long
+
+    return (False, hints) if hints else (True, hints)
+
+
+def coreweave_profile_in_config() -> bool:
+    """Checks if CoreWeave profile is set in config"""
+    conf_path = os.path.expanduser(COREWEAVE_CONFIG_PATH)
+    coreweave_profile_exists = False
+    if os.path.isfile(conf_path):
+        with open(conf_path, 'r', encoding='utf-8') as file:
+            for line in file:
+                if f'[profile {COREWEAVE_PROFILE_NAME}]' in line:
+                    coreweave_profile_exists = True
+                    break
+    return coreweave_profile_exists
+
+
+def coreweave_profile_in_cred() -> bool:
+    """Checks if CoreWeave profile is set in credentials"""
+    cred_path = os.path.expanduser(COREWEAVE_CREDENTIALS_PATH)
+    coreweave_profile_exists = False
+    if os.path.isfile(cred_path):
+        with open(cred_path, 'r', encoding='utf-8') as file:
+            for line in file:
+                if f'[{COREWEAVE_PROFILE_NAME}]' in line:
+                    coreweave_profile_exists = True
+                    break
+    return coreweave_profile_exists
+
+
+def get_credential_file_mounts() -> Dict[str, str]:
+    """Returns credential file mounts for CoreWeave.
+
+    Returns:
+        Dict[str, str]: A dictionary mapping source paths to destination paths
+        for credential files.
+    """
+    coreweave_credential_mounts = {
+        COREWEAVE_CREDENTIALS_PATH: COREWEAVE_CREDENTIALS_PATH,
+        COREWEAVE_CONFIG_PATH: COREWEAVE_CONFIG_PATH
+    }
+    return coreweave_credential_mounts

sky/adaptors/kubernetes.py

@@ -1,4 +1,5 @@
 """Kubernetes adaptors"""
+import functools
 import logging
 import os
 import platform
@@ -162,8 +163,61 @@ def list_kube_config_contexts():
     return kubernetes.config.list_kube_config_contexts(_get_config_file())
 
 
+class ClientWrapper:
+    """Wrapper around the kubernetes API clients.
+
+    This is needed because we cache kubernetes.client.ApiClient and other typed
+    clients (e.g. kubernetes.client.CoreV1Api) and lru_cache.cache_clear() does
+    not call close() on the client to cleanup external resources like
+    semaphores. This decorator wraps the client with __del__ to ensure the
+    external state of kubernetes clients are properly cleaned up on GC.
+    """
+
+    def __init__(self, client):
+        self._client = client
+
+    def __getattr__(self, name):
+        """Delegate to the underlying client"""
+        return getattr(self._client, name)
+
+    def __del__(self):
+        """Clean up the underlying client"""
+        try:
+            real_client = None
+            if isinstance(self._client, kubernetes.client.ApiClient):
+                real_client = self._client
+            elif isinstance(self._client, kubernetes.watch.Watch):
+                real_client = getattr(self._client, '_api_client', None)
+            else:
+                # Otherwise, the client is a typed client, the typed client
+                # is generated by codegen and all of them should have an
+                # 'api_client' attribute referring to the real client.
+                real_client = getattr(self._client, 'api_client', None)
+            if real_client is not None:
+                real_client.close()
+            else:
+                # logger may already be cleaned up during __del__ at shutdown
+                if logger is not None:
+                    logger.debug(f'No client found for {self._client}')
+        except Exception as e:  # pylint: disable=broad-except
+            if logger is not None:
+                logger.debug(f'Error closing Kubernetes client: {e}')
+
+
+def wrap_kubernetes_client(func):
+    """Wraps kubernetes API clients for proper cleanup."""
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        obj = func(*args, **kwargs)
+        return ClientWrapper(obj)
+
+    return wrapper
+
+
 @_api_logging_decorator('urllib3', logging.ERROR)
 @annotations.lru_cache(scope='request')
+@wrap_kubernetes_client
 def core_api(context: Optional[str] = None):
     _load_config(context)
     return kubernetes.client.CoreV1Api()
@@ -171,6 +225,7 @@ def core_api(context: Optional[str] = None):
 
 @_api_logging_decorator('urllib3', logging.ERROR)
 @annotations.lru_cache(scope='request')
+@wrap_kubernetes_client
 def storage_api(context: Optional[str] = None):
     _load_config(context)
     return kubernetes.client.StorageV1Api()
@@ -178,6 +233,7 @@ def storage_api(context: Optional[str] = None):
 
 @_api_logging_decorator('urllib3', logging.ERROR)
 @annotations.lru_cache(scope='request')
+@wrap_kubernetes_client
 def auth_api(context: Optional[str] = None):
     _load_config(context)
     return kubernetes.client.RbacAuthorizationV1Api()
@@ -185,6 +241,7 @@ def auth_api(context: Optional[str] = None):
 
 @_api_logging_decorator('urllib3', logging.ERROR)
 @annotations.lru_cache(scope='request')
+@wrap_kubernetes_client
 def networking_api(context: Optional[str] = None):
     _load_config(context)
     return kubernetes.client.NetworkingV1Api()
@@ -192,6 +249,7 @@ def networking_api(context: Optional[str] = None):
 
 @_api_logging_decorator('urllib3', logging.ERROR)
 @annotations.lru_cache(scope='request')
+@wrap_kubernetes_client
 def custom_objects_api(context: Optional[str] = None):
     _load_config(context)
     return kubernetes.client.CustomObjectsApi()
@@ -199,6 +257,7 @@ def custom_objects_api(context: Optional[str] = None):
 
 @_api_logging_decorator('urllib3', logging.ERROR)
 @annotations.lru_cache(scope='global')
+@wrap_kubernetes_client
 def node_api(context: Optional[str] = None):
     _load_config(context)
     return kubernetes.client.NodeV1Api()
@@ -206,6 +265,7 @@ def node_api(context: Optional[str] = None):
 
 @_api_logging_decorator('urllib3', logging.ERROR)
 @annotations.lru_cache(scope='request')
+@wrap_kubernetes_client
 def apps_api(context: Optional[str] = None):
     _load_config(context)
     return kubernetes.client.AppsV1Api()
@@ -213,6 +273,7 @@ def apps_api(context: Optional[str] = None):
 
 @_api_logging_decorator('urllib3', logging.ERROR)
 @annotations.lru_cache(scope='request')
+@wrap_kubernetes_client
 def batch_api(context: Optional[str] = None):
     _load_config(context)
     return kubernetes.client.BatchV1Api()
@@ -220,6 +281,7 @@ def batch_api(context: Optional[str] = None):
 
 @_api_logging_decorator('urllib3', logging.ERROR)
 @annotations.lru_cache(scope='request')
+@wrap_kubernetes_client
 def api_client(context: Optional[str] = None):
     _load_config(context)
     return kubernetes.client.ApiClient()
@@ -227,6 +289,7 @@ def api_client(context: Optional[str] = None):
 
 @_api_logging_decorator('urllib3', logging.ERROR)
 @annotations.lru_cache(scope='request')
+@wrap_kubernetes_client
 def custom_resources_api(context: Optional[str] = None):
     _load_config(context)
     return kubernetes.client.CustomObjectsApi()
@@ -234,6 +297,7 @@ def custom_resources_api(context: Optional[str] = None):
 
 @_api_logging_decorator('urllib3', logging.ERROR)
 @annotations.lru_cache(scope='request')
+@wrap_kubernetes_client
 def watch(context: Optional[str] = None):
     _load_config(context)
     return kubernetes.watch.Watch()

sky/adaptors/shadeform.py

@@ -0,0 +1,89 @@
+"""Shadeform cloud adaptor."""
+
+import functools
+import socket
+from typing import Any, Dict, List, Optional
+
+import requests
+
+from sky import sky_logging
+from sky.provision.shadeform import shadeform_utils
+from sky.utils import common_utils
+
+logger = sky_logging.init_logger(__name__)
+
+_shadeform_sdk = None
+
+
+def import_package(func):
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        global _shadeform_sdk
+        if _shadeform_sdk is None:
+            try:
+                import shadeform as _shadeform  # pylint: disable=import-outside-toplevel
+                _shadeform_sdk = _shadeform
+            except ImportError:
+                raise ImportError(
+                    'Failed to import dependencies for Shadeform. '
+                    'Try pip install "skypilot[shadeform]"') from None
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+@import_package
+def shadeform():
+    """Return the shadeform package."""
+    return _shadeform_sdk
+
+
+def list_ssh_keys() -> List[Dict[str, Any]]:
+    """List all SSH keys in Shadeform account."""
+    try:
+        response = shadeform_utils.get_ssh_keys()
+        return response.get('ssh_keys', [])
+    except (ValueError, KeyError, requests.exceptions.RequestException) as e:
+        logger.warning(f'Failed to list SSH keys from Shadeform: {e}')
+        return []
+
+
+def add_ssh_key_to_shadeform(public_key: str) -> Optional[str]:
+    """Add SSH key to Shadeform if it doesn't already exist.
+
+    Args:
+        public_key: The SSH public key string.
+
+    Returns:
+        The name of the key if added successfully, None otherwise.
+    """
+    try:
+        # Check if key already exists
+        existing_keys = list_ssh_keys()
+        key_exists = False
+        key_id = None
+        for key in existing_keys:
+            if key.get('public_key') == public_key:
+                key_exists = True
+                key_id = key.get('id')
+                break
+
+        if key_exists:
+            logger.info('SSH key already exists in Shadeform account')
+            return key_id
+
+        # Generate a unique key name
+        hostname = socket.gethostname()
+        key_name = f'skypilot-{hostname}-{common_utils.get_user_hash()[:8]}'
+
+        # Add the key
+        response = shadeform_utils.add_ssh_key(name=key_name,
+                                               public_key=public_key)
+        key_id = response['id']
+        logger.info(f'Added SSH key to Shadeform: {key_name, key_id}')
+        return key_id
+
+    except (ValueError, KeyError, requests.exceptions.RequestException) as e:
+        logger.warning(f'Failed to add SSH key to Shadeform: {e}')
+        return None