skypilot-nightly 1.0.0.dev20250905__py3-none-any.whl → 1.0.0.dev20251203__py3-none-any.whl

sky/__init__.py

@@ -7,7 +7,7 @@ import urllib.request
 from sky.utils import directory_utils
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = 'af1336c82e1e14bd9cd6f27d3dc3d5441a22e83d'
+_SKYPILOT_COMMIT_SHA = '3ff39aba6d4752d5c3b09e3fa7d778cefea39370'
 
 
 def _get_git_commit():
@@ -37,7 +37,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250905'
+__version__ = '1.0.0.dev20251203'
 __root_dir__ = directory_utils.get_sky_dir()
 
 
@@ -122,6 +122,7 @@ from sky.data import StoreType
 from sky.jobs import ManagedJobStatus
 from sky.optimizer import Optimizer
 from sky.resources import Resources
+from sky.server.requests.request_names import AdminPolicyRequestName
 from sky.skylet.job_lib import JobStatus
 from sky.task import Task
 from sky.utils.common import OptimizeTarget
@@ -143,12 +144,15 @@ Kubernetes = clouds.Kubernetes
 K8s = Kubernetes
 OCI = clouds.OCI
 Paperspace = clouds.Paperspace
+PrimeIntellect = clouds.PrimeIntellect
 RunPod = clouds.RunPod
 Vast = clouds.Vast
 Vsphere = clouds.Vsphere
 Fluidstack = clouds.Fluidstack
 Nebius = clouds.Nebius
 Hyperbolic = clouds.Hyperbolic
+Shadeform = clouds.Shadeform
+Seeweb = clouds.Seeweb
 
 __all__ = [
     '__version__',
@@ -162,6 +166,7 @@ __all__ = [
     'Lambda',
     'OCI',
     'Paperspace',
+    'PrimeIntellect',
     'RunPod',
     'Vast',
     'SCP',
@@ -169,6 +174,8 @@ __all__ = [
     'Fluidstack',
     'Nebius',
     'Hyperbolic',
+    'Shadeform',
+    'Seeweb',
     'Optimizer',
     'OptimizeTarget',
     'backends',
@@ -222,6 +229,7 @@ __all__ = [
     'MutatedUserRequest',
     'AdminPolicy',
     'Config',
+    'AdminPolicyRequestName',
     # Registry
     'CLOUD_REGISTRY',
     'JOBS_RECOVERY_STRATEGY_REGISTRY',

sky/adaptors/aws.py

@@ -28,12 +28,14 @@ This is informed by the following boto3 docs:
 
 # pylint: disable=import-outside-toplevel
 
+import functools
 import logging
 import threading
 import time
 import typing
 from typing import Callable, Literal, Optional, TypeVar
 
+from sky import skypilot_config
 from sky.adaptors import common
 from sky.utils import annotations
 from sky.utils import common_utils
@@ -67,17 +69,63 @@ version = 1
 _MAX_ATTEMPT_FOR_CREATION = 5
 
 
-class _ThreadLocalLRUCache(threading.local):
+class _ThreadLocalTTLCache(threading.local):
+    """Thread-local storage for _thread_local_lru_cache decorator."""
 
-    def __init__(self, maxsize=32):
+    def __init__(self, func, maxsize: int, ttl: int):
         super().__init__()
-        self.cache = annotations.lru_cache(scope='request', maxsize=maxsize)
+        self.func = func
+        self.maxsize = maxsize
+        self.ttl = ttl
 
+    def get_cache(self):
+        if not hasattr(self, 'cache'):
+            self.cache = annotations.ttl_cache(scope='request',
+                                               maxsize=self.maxsize,
+                                               ttl=self.ttl,
+                                               timer=time.time)(self.func)
+        return self.cache
 
-def _thread_local_lru_cache(maxsize=32):
-    # Create thread-local storage for the LRU cache
-    local_cache = _ThreadLocalLRUCache(maxsize)
-    return local_cache.cache
+
+def _thread_local_ttl_cache(maxsize=32, ttl=60 * 55):
+    """Thread-local TTL cache decorator.
+
+    Args:
+        maxsize: Maximum size of the cache.
+        ttl: Time to live for the cache in seconds.
+             Default is 55 minutes, a bit less than 1 hour
+             default lifetime of an STS token.
+    """
+
+    def decorator(func):
+        # Create thread-local storage for the LRU cache
+        local_cache = _ThreadLocalTTLCache(func, maxsize, ttl)
+
+        # We can't apply the lru_cache here, because this runs at import time
+        # so we will always have the main thread's cache.
+
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            # We are within the actual function call, which may be on a thread,
+            # so local_cache.cache will return the correct thread-local cache,
+            # which we can now apply and immediately call.
+            return local_cache.get_cache()(*args, **kwargs)
+
+        def cache_info():
+            # Note that this will only give the cache info for the current
+            # thread's cache.
+            return local_cache.get_cache().cache_info()
+
+        def cache_clear():
+            # Note that this will only clear the cache for the current thread.
+            local_cache.get_cache().cache_clear()
+
+        wrapper.cache_info = cache_info  # type: ignore[attr-defined]
+        wrapper.cache_clear = cache_clear  # type: ignore[attr-defined]
+
+        return wrapper
+
+    return decorator
 
 
 def _assert_kwargs_builtin_type(kwargs):
@@ -119,12 +167,27 @@ def _create_aws_object(creation_fn_or_cls: Callable[[], T],
                         f'{common_utils.format_exception(e)}.')
 
 
-# The LRU cache needs to be thread-local to avoid multiple threads sharing the
+def get_workspace_profile() -> Optional[str]:
+    """Get AWS profile name from workspace config."""
+    return skypilot_config.get_workspace_cloud('aws').get('profile', None)
+
+
+# The TTL cache needs to be thread-local to avoid multiple threads sharing the
 # same session object, which is not guaranteed to be thread-safe.
-@_thread_local_lru_cache()
-def session(check_credentials: bool = True):
-    """Create an AWS session."""
-    s = _create_aws_object(boto3.session.Session, 'session')
+@_thread_local_ttl_cache()
+def session(check_credentials: bool = True, profile: Optional[str] = None):
+    """Create an AWS session.
+
+    Args:
+        check_credentials: Whether to check if credentials are available.
+        profile: AWS profile name to use. If None, uses default credentials.
+    """
+    if profile is not None:
+        logger.debug(f'Using AWS profile \'{profile}\'.')
+        s = _create_aws_object(
+            lambda: boto3.session.Session(profile_name=profile), 'session')
+    else:
+        s = _create_aws_object(boto3.session.Session, 'session')
     if check_credentials and s.get_credentials() is None:
         # s.get_credentials() can be None if there are actually no credentials,
         # or if we fail to get credentials from IMDS (e.g. due to throttling).
@@ -180,13 +243,14 @@ def resource(service_name: str, **kwargs):
         kwargs['config'] = config
 
     check_credentials = kwargs.pop('check_credentials', True)
+    profile = get_workspace_profile()
 
     # Need to use the client retrieved from the per-thread session to avoid
     # thread-safety issues (Directly creating the client with boto3.resource()
     # is not thread-safe). Reference: https://stackoverflow.com/a/59635814
     return _create_aws_object(
-        lambda: session(check_credentials=check_credentials).resource(
-            service_name, **kwargs), 'resource')
+        lambda: session(check_credentials=check_credentials, profile=profile).
+        resource(service_name, **kwargs), 'resource')
 
 
 # New typing overloads can be added as needed.
@@ -221,14 +285,15 @@ def client(service_name: str, **kwargs):
     _assert_kwargs_builtin_type(kwargs)
 
     check_credentials = kwargs.pop('check_credentials', True)
+    profile = get_workspace_profile()
 
     # Need to use the client retrieved from the per-thread session to avoid
     # thread-safety issues (Directly creating the client with boto3.client() is
     # not thread-safe). Reference: https://stackoverflow.com/a/59635814
 
     return _create_aws_object(
-        lambda: session(check_credentials=check_credentials).client(
-            service_name, **kwargs), 'client')
+        lambda: session(check_credentials=check_credentials, profile=profile).
+        client(service_name, **kwargs), 'client')
 
 
 @common.load_lazy_modules(modules=_LAZY_MODULES)

sky/adaptors/common.py

@@ -1,9 +1,10 @@
 """Lazy import for modules to avoid import error when not used."""
+from importlib import util as importlib_util
 import functools
 import importlib
 import threading
 import types
-from typing import Any, Callable, Optional, Tuple
+from typing import Any, Callable, List, Optional, Tuple
 
 
 class LazyImport(types.ModuleType):
@@ -27,7 +28,7 @@ class LazyImport(types.ModuleType):
                  import_error_message: Optional[str] = None,
                  set_loggers: Optional[Callable] = None):
         self._module_name = module_name
-        self._module = None
+        self._module: Optional[types.ModuleType] = None
         self._import_error_message = import_error_message
         self._set_loggers = set_loggers
         self._lock = threading.RLock()
@@ -78,3 +79,25 @@ def load_lazy_modules(modules: Tuple[LazyImport, ...]):
         return wrapper
 
     return decorator
+
+
+def can_import_modules(module_names: List[str]) -> bool:
+    """ module availability without actually importing it to
+    save memory footprint.
+
+    Args:
+        module_names: List[str], the names of the modules to check.
+
+    Returns:
+        True if all modules are available, False otherwise.
+        If a module exists in sys.modules, but is set to None,
+        then it is considered as not available.
+    """
+    try:
+        for module_name in module_names:
+            module_spec = importlib_util.find_spec(module_name)
+            if module_spec is None:
+                return False
+        return True
+    except ValueError:
+        return False

sky/adaptors/coreweave.py

@@ -0,0 +1,278 @@
+"""CoreWeave cloud adaptor."""
+
+import configparser
+import contextlib
+import os
+import threading
+from typing import Dict, Optional, Tuple
+
+from sky import exceptions
+from sky import sky_logging
+from sky.adaptors import common
+from sky.clouds import cloud
+from sky.utils import annotations
+from sky.utils import ux_utils
+
+logger = sky_logging.init_logger(__name__)
+
+COREWEAVE_PROFILE_NAME = 'cw'
+COREWEAVE_CREDENTIALS_PATH = '~/.coreweave/cw.credentials'
+COREWEAVE_CONFIG_PATH = '~/.coreweave/cw.config'
+NAME = 'CoreWeave'
+DEFAULT_REGION = 'US-EAST-01A'
+_DEFAULT_ENDPOINT = 'https://cwobject.com'
+_INDENT_PREFIX = '    '
+
+_IMPORT_ERROR_MESSAGE = ('Failed to import dependencies for CoreWeave.'
+                         'Try pip install "skypilot[coreweave]"')
+
+boto3 = common.LazyImport('boto3', import_error_message=_IMPORT_ERROR_MESSAGE)
+botocore = common.LazyImport('botocore',
+                             import_error_message=_IMPORT_ERROR_MESSAGE)
+
+_LAZY_MODULES = (boto3, botocore)
+_session_creation_lock = threading.RLock()
+
+
+@contextlib.contextmanager
+def _load_cw_credentials_env():
+    """Context manager to temporarily change the AWS credentials file path."""
+    prev_credentials_path = os.environ.get('AWS_SHARED_CREDENTIALS_FILE')
+    prev_config_path = os.environ.get('AWS_CONFIG_FILE')
+    os.environ['AWS_SHARED_CREDENTIALS_FILE'] = COREWEAVE_CREDENTIALS_PATH
+    os.environ['AWS_CONFIG_FILE'] = COREWEAVE_CONFIG_PATH
+    try:
+        yield
+    finally:
+        if prev_credentials_path is None:
+            del os.environ['AWS_SHARED_CREDENTIALS_FILE']
+        else:
+            os.environ['AWS_SHARED_CREDENTIALS_FILE'] = prev_credentials_path
+        if prev_config_path is None:
+            del os.environ['AWS_CONFIG_FILE']
+        else:
+            os.environ['AWS_CONFIG_FILE'] = prev_config_path
+
+
+def get_coreweave_credentials(boto3_session):
+    """Gets the CoreWeave credentials from the boto3 session object.
+
+    Args:
+        boto3_session: The boto3 session object.
+    Returns:
+        botocore.credentials.ReadOnlyCredentials object with the CoreWeave
+        credentials.
+    """
+    with _load_cw_credentials_env():
+        coreweave_credentials = boto3_session.get_credentials()
+        if coreweave_credentials is None:
+            with ux_utils.print_exception_no_traceback():
+                raise ValueError('CoreWeave credentials not found. Run '
+                                 '`sky check` to verify credentials are '
+                                 'correctly set up.')
+        return coreweave_credentials.get_frozen_credentials()
+
+
+@annotations.lru_cache(scope='global')
+def session():
+    """Create an AWS session for CoreWeave."""
+    # Creating the session object is not thread-safe for boto3,
+    # so we add a reentrant lock to synchronize the session creation.
+    # Reference: https://github.com/boto/boto3/issues/1592
+    # However, the session object itself is thread-safe, so we are
+    # able to use lru_cache() to cache the session object.
+    with _session_creation_lock:
+        with _load_cw_credentials_env():
+            session_ = boto3.session.Session(
+                profile_name=COREWEAVE_PROFILE_NAME)
+        return session_
+
+
+@annotations.lru_cache(scope='global')
+def resource(resource_name: str, **kwargs):
+    """Create a CoreWeave resource.
+
+    Args:
+        resource_name: CoreWeave resource name (e.g., 's3').
+        kwargs: Other options.
+    """
+    # Need to use the resource retrieved from the per-thread session
+    # to avoid thread-safety issues (Directly creating the client
+    # with boto3.resource() is not thread-safe).
+    # Reference: https://stackoverflow.com/a/59635814
+
+    session_ = session()
+    coreweave_credentials = get_coreweave_credentials(session_)
+    endpoint = get_endpoint()
+
+    return session_.resource(
+        resource_name,
+        endpoint_url=endpoint,
+        aws_access_key_id=coreweave_credentials.access_key,
+        aws_secret_access_key=coreweave_credentials.secret_key,
+        region_name='auto',
+        config=botocore.config.Config(s3={'addressing_style': 'virtual'}),
+        **kwargs)
+
+
+@annotations.lru_cache(scope='global')
+def client(service_name: str):
+    """Create CoreWeave client of a certain service.
+
+    Args:
+        service_name: CoreWeave service name (e.g., 's3').
+    """
+    # Need to use the client retrieved from the per-thread session
+    # to avoid thread-safety issues (Directly creating the client
+    # with boto3.client() is not thread-safe).
+    # Reference: https://stackoverflow.com/a/59635814
+
+    session_ = session()
+    coreweave_credentials = get_coreweave_credentials(session_)
+    endpoint = get_endpoint()
+
+    return session_.client(
+        service_name,
+        endpoint_url=endpoint,
+        aws_access_key_id=coreweave_credentials.access_key,
+        aws_secret_access_key=coreweave_credentials.secret_key,
+        region_name='auto',
+        config=botocore.config.Config(s3={'addressing_style': 'virtual'}),
+    )
+
+
+@common.load_lazy_modules(_LAZY_MODULES)
+def botocore_exceptions():
+    """AWS botocore exception."""
+    # pylint: disable=import-outside-toplevel
+    from botocore import exceptions as boto_exceptions
+    return boto_exceptions
+
+
+def get_endpoint():
+    """Parse the COREWEAVE_CONFIG_PATH to get the endpoint_url.
+
+    The config file is an AWS-style config file with format:
+        [profile cw]
+        endpoint_url = https://cwobject.com
+        s3 =
+            addressing_style = virtual
+
+    Returns:
+        str: The endpoint URL from the config file, or the default endpoint
+             if the file doesn't exist or doesn't contain the endpoint_url.
+    """
+    config_path = os.path.expanduser(COREWEAVE_CONFIG_PATH)
+    if not os.path.isfile(config_path):
+        return _DEFAULT_ENDPOINT
+
+    try:
+        config = configparser.ConfigParser()
+        config.read(config_path)
+
+        # Try to get endpoint_url from [profile cw] section
+        profile_section = f'profile {COREWEAVE_PROFILE_NAME}'
+        if config.has_section(profile_section):
+            if config.has_option(profile_section, 'endpoint_url'):
+                endpoint = config.get(profile_section, 'endpoint_url')
+                return endpoint.strip()
+    except (configparser.Error, OSError) as e:
+        logger.warning(f'Failed to parse CoreWeave config file: {e}. '
+                       f'Using default endpoint: {_DEFAULT_ENDPOINT}')
+
+    return _DEFAULT_ENDPOINT
+
+
+def check_credentials(
+        cloud_capability: cloud.CloudCapability) -> Tuple[bool, Optional[str]]:
+    if cloud_capability == cloud.CloudCapability.STORAGE:
+        return check_storage_credentials()
+    else:
+        raise exceptions.NotSupportedError(
+            f'{NAME} does not support {cloud_capability}.')
+
+
+def check_storage_credentials() -> Tuple[bool, Optional[str]]:
+    """Checks if the user has access credentials to CoreWeave Object Storage.
+
+    Returns:
+        A tuple of a boolean value and a hint message where the bool
+        is True when both credentials needed for CoreWeave storage is set.
+        It is False when either of those are not set, which would hint with a
+        string on unset credential.
+    """
+    hints = None
+    profile_in_cred = coreweave_profile_in_cred()
+    profile_in_config = coreweave_profile_in_config()
+
+    if not profile_in_cred:
+        hints = (f'[{COREWEAVE_PROFILE_NAME}] profile is not set in '
+                 f'{COREWEAVE_CREDENTIALS_PATH}.')
+    if not profile_in_config:
+        if hints:
+            hints += ' Additionally, '
+        else:
+            hints = ''
+        hints += (f'[{COREWEAVE_PROFILE_NAME}] profile is not set in '
+                  f'{COREWEAVE_CONFIG_PATH}.')
+
+    if hints:
+        hints += ' Run the following commands:'
+        if not profile_in_cred:
+            hints += f'\n{_INDENT_PREFIX}  $ pip install boto3'
+            hints += (f'\n{_INDENT_PREFIX}  $ AWS_SHARED_CREDENTIALS_FILE='
+                      f'{COREWEAVE_CREDENTIALS_PATH} aws configure --profile '
+                      f'{COREWEAVE_PROFILE_NAME}')
+        if not profile_in_config:
+            hints += (f'\n{_INDENT_PREFIX}  $ AWS_CONFIG_FILE='
+                      f'{COREWEAVE_CONFIG_PATH} aws configure set endpoint_url'
+                      f' <ENDPOINT_URL> --profile '
+                      f'{COREWEAVE_PROFILE_NAME}')
+            hints += (f'\n{_INDENT_PREFIX}  $ AWS_CONFIG_FILE='
+                      f'{COREWEAVE_CONFIG_PATH} aws configure set '
+                      f's3.addressing_style virtual --profile '
+                      f'{COREWEAVE_PROFILE_NAME}')
+        hints += f'\n{_INDENT_PREFIX}For more info: '
+        hints += 'https://docs.coreweave.com/docs/products/storage/object-storage/get-started-caios'  # pylint: disable=line-too-long
+
+    return (False, hints) if hints else (True, hints)
+
+
+def coreweave_profile_in_config() -> bool:
+    """Checks if CoreWeave profile is set in config"""
+    conf_path = os.path.expanduser(COREWEAVE_CONFIG_PATH)
+    coreweave_profile_exists = False
+    if os.path.isfile(conf_path):
+        with open(conf_path, 'r', encoding='utf-8') as file:
+            for line in file:
+                if f'[profile {COREWEAVE_PROFILE_NAME}]' in line:
+                    coreweave_profile_exists = True
+                    break
+    return coreweave_profile_exists
+
+
+def coreweave_profile_in_cred() -> bool:
+    """Checks if CoreWeave profile is set in credentials"""
+    cred_path = os.path.expanduser(COREWEAVE_CREDENTIALS_PATH)
+    coreweave_profile_exists = False
+    if os.path.isfile(cred_path):
+        with open(cred_path, 'r', encoding='utf-8') as file:
+            for line in file:
+                if f'[{COREWEAVE_PROFILE_NAME}]' in line:
+                    coreweave_profile_exists = True
+                    break
+    return coreweave_profile_exists
+
+
+def get_credential_file_mounts() -> Dict[str, str]:
+    """Returns credential file mounts for CoreWeave.
+
+    Returns:
+        Dict[str, str]: A dictionary mapping source paths to destination paths
+        for credential files.
+    """
+    coreweave_credential_mounts = {
+        COREWEAVE_CREDENTIALS_PATH: COREWEAVE_CREDENTIALS_PATH,
+        COREWEAVE_CONFIG_PATH: COREWEAVE_CONFIG_PATH
+    }
+    return coreweave_credential_mounts

sky/adaptors/do.py

@@ -1,7 +1,5 @@
 """Digital Ocean cloud adaptors"""
 
-# pylint: disable=import-outside-toplevel
-
 from sky.adaptors import common
 
 _IMPORT_ERROR_MESSAGE = ('Failed to import dependencies for DO. '
@@ -16,5 +14,13 @@ _LAZY_MODULES = (pydo, azure)
 @common.load_lazy_modules(modules=_LAZY_MODULES)
 def exceptions():
     """Azure exceptions."""
+    # pylint: disable=import-outside-toplevel
     from azure.core import exceptions as azure_exceptions
     return azure_exceptions
+
+
+def check_exceptions_dependencies_installed():
+    """Check if the azure.core.exceptions module is installed."""
+    if not common.can_import_modules(['azure.core.exceptions']):
+        return False, _IMPORT_ERROR_MESSAGE
+    return True, None

sky/adaptors/gcp.py

@@ -2,9 +2,20 @@
 
 # pylint: disable=import-outside-toplevel
 import json
+import warnings
 
 from sky.adaptors import common
 
+# Suppress FutureWarning from google.api_core about Python 3.10 support ending.
+# This warning is informational and does not affect functionality.
+# Reference: https://github.com/skypilot-org/skypilot/issues/7886
+warnings.filterwarnings(
+    'ignore',
+    category=FutureWarning,
+    message=
+    r'.*You are using a Python version.*which Google will stop supporting.*',
+)
+
 _IMPORT_ERROR_MESSAGE = ('Failed to import dependencies for GCP. '
                          'Try pip install "skypilot[gcp]"')
 googleapiclient = common.LazyImport('googleapiclient',

sky/adaptors/ibm.py

@@ -28,6 +28,9 @@ requests = common.LazyImport('requests',
                              import_error_message=_IMPORT_ERROR_MESSAGE)
 yaml = common.LazyImport('yaml', import_error_message=_IMPORT_ERROR_MESSAGE)
 
+# Global process lock for thread-safe boto3 operations
+global_process_lock = None
+
 
 def read_credential_file():
     try:
@@ -152,9 +155,9 @@ def _get_global_process_lock():
       already initialized.
     Necessary when process are spawned without a shared lock.
     """
-    global global_process_lock  # pylint: disable=global-variable-undefined
+    global global_process_lock
 
-    if 'global_process_lock' not in globals():
+    if global_process_lock is None:
         global_process_lock = multiprocessing.Lock()
 
     return global_process_lock