skypilot-nightly 1.0.0.dev20250509__py3-none-any.whl → 1.0.0.dev20251107__py3-none-any.whl

sky/__init__.py

@@ -4,8 +4,10 @@ import subprocess
 from typing import Optional
 import urllib.request
 
+from sky.utils import directory_utils
+
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '65783882c8476dfb543ba54976382281df61867c'
+_SKYPILOT_COMMIT_SHA = 'd7530d48bc1a331b5644bd6fbbc51eaebf1432f3'
 
 
 def _get_git_commit():
@@ -35,8 +37,8 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250509'
-__root_dir__ = os.path.dirname(os.path.abspath(__file__))
+__version__ = '1.0.0.dev20251107'
+__root_dir__ = directory_utils.get_sky_dir()
 
 
 # ---------------------- Proxy Configuration ---------------------- #
@@ -81,13 +83,14 @@ _set_http_proxy_env_vars()
 # Keep this order to avoid cyclic imports
 # pylint: disable=wrong-import-position
 from sky import backends
-from sky import benchmark
 from sky import clouds
 from sky.admin_policy import AdminPolicy
 from sky.admin_policy import MutatedUserRequest
 from sky.admin_policy import UserRequest
+from sky.catalog import list_accelerators
 from sky.client.sdk import api_cancel
 from sky.client.sdk import api_info
+from sky.client.sdk import api_login
 from sky.client.sdk import api_server_logs
 from sky.client.sdk import api_start
 from sky.client.sdk import api_status
@@ -97,12 +100,14 @@ from sky.client.sdk import cancel
 from sky.client.sdk import cost_report
 from sky.client.sdk import down
 from sky.client.sdk import download_logs
+from sky.client.sdk import endpoints
 from sky.client.sdk import exec  # pylint: disable=redefined-builtin
 from sky.client.sdk import get
 from sky.client.sdk import job_status
 from sky.client.sdk import launch
 from sky.client.sdk import optimize
 from sky.client.sdk import queue
+from sky.client.sdk import reload_config
 from sky.client.sdk import start
 from sky.client.sdk import status
 from sky.client.sdk import stop
@@ -110,7 +115,6 @@ from sky.client.sdk import storage_delete
 from sky.client.sdk import storage_ls
 from sky.client.sdk import stream_and_get
 from sky.client.sdk import tail_logs
-from sky.clouds.service_catalog import list_accelerators
 from sky.dag import Dag
 from sky.data import Storage
 from sky.data import StorageMode
@@ -118,6 +122,7 @@ from sky.data import StoreType
 from sky.jobs import ManagedJobStatus
 from sky.optimizer import Optimizer
 from sky.resources import Resources
+from sky.server.requests.request_names import AdminPolicyRequestName
 from sky.skylet.job_lib import JobStatus
 from sky.task import Task
 from sky.utils.common import OptimizeTarget
@@ -139,11 +144,15 @@ Kubernetes = clouds.Kubernetes
 K8s = Kubernetes
 OCI = clouds.OCI
 Paperspace = clouds.Paperspace
+PrimeIntellect = clouds.PrimeIntellect
 RunPod = clouds.RunPod
 Vast = clouds.Vast
 Vsphere = clouds.Vsphere
 Fluidstack = clouds.Fluidstack
 Nebius = clouds.Nebius
+Hyperbolic = clouds.Hyperbolic
+Shadeform = clouds.Shadeform
+Seeweb = clouds.Seeweb
 
 __all__ = [
     '__version__',
@@ -157,16 +166,19 @@ __all__ = [
     'Lambda',
     'OCI',
     'Paperspace',
+    'PrimeIntellect',
     'RunPod',
     'Vast',
     'SCP',
     'Vsphere',
     'Fluidstack',
     'Nebius',
+    'Hyperbolic',
+    'Shadeform',
+    'Seeweb',
     'Optimizer',
     'OptimizeTarget',
     'backends',
-    'benchmark',
     'list_accelerators',
     '__root_dir__',
     'Storage',
@@ -184,6 +196,7 @@ __all__ = [
     'optimize',
     'launch',
     'exec',
+    'reload_config',
     # core APIs
     'status',
     'start',
@@ -191,6 +204,7 @@ __all__ = [
     'down',
     'autostop',
     'cost_report',
+    'endpoints',
     # core APIs Job Management
     'queue',
     'cancel',
@@ -206,6 +220,7 @@ __all__ = [
     'api_status',
     'api_cancel',
     'api_info',
+    'api_login',
     'api_start',
     'api_stop',
     'api_server_logs',
@@ -214,6 +229,7 @@ __all__ = [
     'MutatedUserRequest',
     'AdminPolicy',
     'Config',
+    'AdminPolicyRequestName',
     # Registry
     'CLOUD_REGISTRY',
     'JOBS_RECOVERY_STRATEGY_REGISTRY',

sky/adaptors/aws.py

@@ -34,6 +34,7 @@ import time
 import typing
 from typing import Callable, Literal, Optional, TypeVar
 
+from sky import skypilot_config
 from sky.adaptors import common
 from sky.utils import annotations
 from sky.utils import common_utils
@@ -119,12 +120,27 @@ def _create_aws_object(creation_fn_or_cls: Callable[[], T],
                         f'{common_utils.format_exception(e)}.')
 
 
+def get_workspace_profile() -> Optional[str]:
+    """Get AWS profile name from workspace config."""
+    return skypilot_config.get_workspace_cloud('aws').get('profile', None)
+
+
 # The LRU cache needs to be thread-local to avoid multiple threads sharing the
 # same session object, which is not guaranteed to be thread-safe.
 @_thread_local_lru_cache()
-def session(check_credentials: bool = True):
-    """Create an AWS session."""
-    s = _create_aws_object(boto3.session.Session, 'session')
+def session(check_credentials: bool = True, profile: Optional[str] = None):
+    """Create an AWS session.
+
+    Args:
+        check_credentials: Whether to check if credentials are available.
+        profile: AWS profile name to use. If None, uses default credentials.
+    """
+    if profile is not None:
+        logger.debug(f'Using AWS profile \'{profile}\'.')
+        s = _create_aws_object(
+            lambda: boto3.session.Session(profile_name=profile), 'session')
+    else:
+        s = _create_aws_object(boto3.session.Session, 'session')
     if check_credentials and s.get_credentials() is None:
         # s.get_credentials() can be None if there are actually no credentials,
         # or if we fail to get credentials from IMDS (e.g. due to throttling).
@@ -180,13 +196,14 @@ def resource(service_name: str, **kwargs):
         kwargs['config'] = config
 
     check_credentials = kwargs.pop('check_credentials', True)
+    profile = get_workspace_profile()
 
     # Need to use the client retrieved from the per-thread session to avoid
     # thread-safety issues (Directly creating the client with boto3.resource()
     # is not thread-safe). Reference: https://stackoverflow.com/a/59635814
     return _create_aws_object(
-        lambda: session(check_credentials=check_credentials).resource(
-            service_name, **kwargs), 'resource')
+        lambda: session(check_credentials=check_credentials, profile=profile).
+        resource(service_name, **kwargs), 'resource')
 
 
 # New typing overloads can be added as needed.
@@ -221,14 +238,15 @@ def client(service_name: str, **kwargs):
     _assert_kwargs_builtin_type(kwargs)
 
     check_credentials = kwargs.pop('check_credentials', True)
+    profile = get_workspace_profile()
 
     # Need to use the client retrieved from the per-thread session to avoid
     # thread-safety issues (Directly creating the client with boto3.client() is
     # not thread-safe). Reference: https://stackoverflow.com/a/59635814
 
     return _create_aws_object(
-        lambda: session(check_credentials=check_credentials).client(
-            service_name, **kwargs), 'client')
+        lambda: session(check_credentials=check_credentials, profile=profile).
+        client(service_name, **kwargs), 'client')
 
 
 @common.load_lazy_modules(modules=_LAZY_MODULES)

sky/adaptors/common.py

@@ -1,9 +1,10 @@
 """Lazy import for modules to avoid import error when not used."""
+from importlib import util as importlib_util
 import functools
 import importlib
 import threading
 import types
-from typing import Any, Callable, Optional, Tuple
+from typing import Any, Callable, List, Optional, Tuple
 
 
 class LazyImport(types.ModuleType):
@@ -78,3 +79,25 @@ def load_lazy_modules(modules: Tuple[LazyImport, ...]):
         return wrapper
 
     return decorator
+
+
+def can_import_modules(module_names: List[str]) -> bool:
+    """ module availability without actually importing it to
+    save memory footprint.
+
+    Args:
+        module_names: List[str], the names of the modules to check.
+
+    Returns:
+        True if all modules are available, False otherwise.
+        If a module exists in sys.modules, but is set to None,
+        then it is considered as not available.
+    """
+    try:
+        for module_name in module_names:
+            module_spec = importlib_util.find_spec(module_name)
+            if module_spec is None:
+                return False
+        return True
+    except ValueError:
+        return False

sky/adaptors/coreweave.py

@@ -0,0 +1,278 @@
+"""CoreWeave cloud adaptor."""
+
+import configparser
+import contextlib
+import os
+import threading
+from typing import Dict, Optional, Tuple
+
+from sky import exceptions
+from sky import sky_logging
+from sky.adaptors import common
+from sky.clouds import cloud
+from sky.utils import annotations
+from sky.utils import ux_utils
+
+logger = sky_logging.init_logger(__name__)
+
+COREWEAVE_PROFILE_NAME = 'cw'
+COREWEAVE_CREDENTIALS_PATH = '~/.coreweave/cw.credentials'
+COREWEAVE_CONFIG_PATH = '~/.coreweave/cw.config'
+NAME = 'CoreWeave'
+DEFAULT_REGION = 'US-EAST-01A'
+_DEFAULT_ENDPOINT = 'https://cwobject.com'
+_INDENT_PREFIX = '    '
+
+_IMPORT_ERROR_MESSAGE = ('Failed to import dependencies for CoreWeave.'
+                         'Try pip install "skypilot[coreweave]"')
+
+boto3 = common.LazyImport('boto3', import_error_message=_IMPORT_ERROR_MESSAGE)
+botocore = common.LazyImport('botocore',
+                             import_error_message=_IMPORT_ERROR_MESSAGE)
+
+_LAZY_MODULES = (boto3, botocore)
+_session_creation_lock = threading.RLock()
+
+
+@contextlib.contextmanager
+def _load_cw_credentials_env():
+    """Context manager to temporarily change the AWS credentials file path."""
+    prev_credentials_path = os.environ.get('AWS_SHARED_CREDENTIALS_FILE')
+    prev_config_path = os.environ.get('AWS_CONFIG_FILE')
+    os.environ['AWS_SHARED_CREDENTIALS_FILE'] = COREWEAVE_CREDENTIALS_PATH
+    os.environ['AWS_CONFIG_FILE'] = COREWEAVE_CONFIG_PATH
+    try:
+        yield
+    finally:
+        if prev_credentials_path is None:
+            del os.environ['AWS_SHARED_CREDENTIALS_FILE']
+        else:
+            os.environ['AWS_SHARED_CREDENTIALS_FILE'] = prev_credentials_path
+        if prev_config_path is None:
+            del os.environ['AWS_CONFIG_FILE']
+        else:
+            os.environ['AWS_CONFIG_FILE'] = prev_config_path
+
+
+def get_coreweave_credentials(boto3_session):
+    """Gets the CoreWeave credentials from the boto3 session object.
+
+    Args:
+        boto3_session: The boto3 session object.
+    Returns:
+        botocore.credentials.ReadOnlyCredentials object with the CoreWeave
+        credentials.
+    """
+    with _load_cw_credentials_env():
+        coreweave_credentials = boto3_session.get_credentials()
+        if coreweave_credentials is None:
+            with ux_utils.print_exception_no_traceback():
+                raise ValueError('CoreWeave credentials not found. Run '
+                                 '`sky check` to verify credentials are '
+                                 'correctly set up.')
+        return coreweave_credentials.get_frozen_credentials()
+
+
+@annotations.lru_cache(scope='global')
+def session():
+    """Create an AWS session for CoreWeave."""
+    # Creating the session object is not thread-safe for boto3,
+    # so we add a reentrant lock to synchronize the session creation.
+    # Reference: https://github.com/boto/boto3/issues/1592
+    # However, the session object itself is thread-safe, so we are
+    # able to use lru_cache() to cache the session object.
+    with _session_creation_lock:
+        with _load_cw_credentials_env():
+            session_ = boto3.session.Session(
+                profile_name=COREWEAVE_PROFILE_NAME)
+        return session_
+
+
+@annotations.lru_cache(scope='global')
+def resource(resource_name: str, **kwargs):
+    """Create a CoreWeave resource.
+
+    Args:
+        resource_name: CoreWeave resource name (e.g., 's3').
+        kwargs: Other options.
+    """
+    # Need to use the resource retrieved from the per-thread session
+    # to avoid thread-safety issues (Directly creating the client
+    # with boto3.resource() is not thread-safe).
+    # Reference: https://stackoverflow.com/a/59635814
+
+    session_ = session()
+    coreweave_credentials = get_coreweave_credentials(session_)
+    endpoint = get_endpoint()
+
+    return session_.resource(
+        resource_name,
+        endpoint_url=endpoint,
+        aws_access_key_id=coreweave_credentials.access_key,
+        aws_secret_access_key=coreweave_credentials.secret_key,
+        region_name='auto',
+        config=botocore.config.Config(s3={'addressing_style': 'virtual'}),
+        **kwargs)
+
+
+@annotations.lru_cache(scope='global')
+def client(service_name: str):
+    """Create CoreWeave client of a certain service.
+
+    Args:
+        service_name: CoreWeave service name (e.g., 's3').
+    """
+    # Need to use the client retrieved from the per-thread session
+    # to avoid thread-safety issues (Directly creating the client
+    # with boto3.client() is not thread-safe).
+    # Reference: https://stackoverflow.com/a/59635814
+
+    session_ = session()
+    coreweave_credentials = get_coreweave_credentials(session_)
+    endpoint = get_endpoint()
+
+    return session_.client(
+        service_name,
+        endpoint_url=endpoint,
+        aws_access_key_id=coreweave_credentials.access_key,
+        aws_secret_access_key=coreweave_credentials.secret_key,
+        region_name='auto',
+        config=botocore.config.Config(s3={'addressing_style': 'virtual'}),
+    )
+
+
+@common.load_lazy_modules(_LAZY_MODULES)
+def botocore_exceptions():
+    """AWS botocore exception."""
+    # pylint: disable=import-outside-toplevel
+    from botocore import exceptions as boto_exceptions
+    return boto_exceptions
+
+
+def get_endpoint():
+    """Parse the COREWEAVE_CONFIG_PATH to get the endpoint_url.
+
+    The config file is an AWS-style config file with format:
+        [profile cw]
+        endpoint_url = https://cwobject.com
+        s3 =
+            addressing_style = virtual
+
+    Returns:
+        str: The endpoint URL from the config file, or the default endpoint
+             if the file doesn't exist or doesn't contain the endpoint_url.
+    """
+    config_path = os.path.expanduser(COREWEAVE_CONFIG_PATH)
+    if not os.path.isfile(config_path):
+        return _DEFAULT_ENDPOINT
+
+    try:
+        config = configparser.ConfigParser()
+        config.read(config_path)
+
+        # Try to get endpoint_url from [profile cw] section
+        profile_section = f'profile {COREWEAVE_PROFILE_NAME}'
+        if config.has_section(profile_section):
+            if config.has_option(profile_section, 'endpoint_url'):
+                endpoint = config.get(profile_section, 'endpoint_url')
+                return endpoint.strip()
+    except (configparser.Error, OSError) as e:
+        logger.warning(f'Failed to parse CoreWeave config file: {e}. '
+                       f'Using default endpoint: {_DEFAULT_ENDPOINT}')
+
+    return _DEFAULT_ENDPOINT
+
+
+def check_credentials(
+        cloud_capability: cloud.CloudCapability) -> Tuple[bool, Optional[str]]:
+    if cloud_capability == cloud.CloudCapability.STORAGE:
+        return check_storage_credentials()
+    else:
+        raise exceptions.NotSupportedError(
+            f'{NAME} does not support {cloud_capability}.')
+
+
+def check_storage_credentials() -> Tuple[bool, Optional[str]]:
+    """Checks if the user has access credentials to CoreWeave Object Storage.
+
+    Returns:
+        A tuple of a boolean value and a hint message where the bool
+        is True when both credentials needed for CoreWeave storage is set.
+        It is False when either of those are not set, which would hint with a
+        string on unset credential.
+    """
+    hints = None
+    profile_in_cred = coreweave_profile_in_cred()
+    profile_in_config = coreweave_profile_in_config()
+
+    if not profile_in_cred:
+        hints = (f'[{COREWEAVE_PROFILE_NAME}] profile is not set in '
+                 f'{COREWEAVE_CREDENTIALS_PATH}.')
+    if not profile_in_config:
+        if hints:
+            hints += ' Additionally, '
+        else:
+            hints = ''
+        hints += (f'[{COREWEAVE_PROFILE_NAME}] profile is not set in '
+                  f'{COREWEAVE_CONFIG_PATH}.')
+
+    if hints:
+        hints += ' Run the following commands:'
+        if not profile_in_cred:
+            hints += f'\n{_INDENT_PREFIX}  $ pip install boto3'
+            hints += (f'\n{_INDENT_PREFIX}  $ AWS_SHARED_CREDENTIALS_FILE='
+                      f'{COREWEAVE_CREDENTIALS_PATH} aws configure --profile '
+                      f'{COREWEAVE_PROFILE_NAME}')
+        if not profile_in_config:
+            hints += (f'\n{_INDENT_PREFIX}  $ AWS_CONFIG_FILE='
+                      f'{COREWEAVE_CONFIG_PATH} aws configure set endpoint_url'
+                      f' <ENDPOINT_URL> --profile '
+                      f'{COREWEAVE_PROFILE_NAME}')
+            hints += (f'\n{_INDENT_PREFIX}  $ AWS_CONFIG_FILE='
+                      f'{COREWEAVE_CONFIG_PATH} aws configure set '
+                      f's3.addressing_style virtual --profile '
+                      f'{COREWEAVE_PROFILE_NAME}')
+        hints += f'\n{_INDENT_PREFIX}For more info: '
+        hints += 'https://docs.coreweave.com/docs/products/storage/object-storage/get-started-caios'  # pylint: disable=line-too-long
+
+    return (False, hints) if hints else (True, hints)
+
+
+def coreweave_profile_in_config() -> bool:
+    """Checks if CoreWeave profile is set in config"""
+    conf_path = os.path.expanduser(COREWEAVE_CONFIG_PATH)
+    coreweave_profile_exists = False
+    if os.path.isfile(conf_path):
+        with open(conf_path, 'r', encoding='utf-8') as file:
+            for line in file:
+                if f'[profile {COREWEAVE_PROFILE_NAME}]' in line:
+                    coreweave_profile_exists = True
+                    break
+    return coreweave_profile_exists
+
+
+def coreweave_profile_in_cred() -> bool:
+    """Checks if CoreWeave profile is set in credentials"""
+    cred_path = os.path.expanduser(COREWEAVE_CREDENTIALS_PATH)
+    coreweave_profile_exists = False
+    if os.path.isfile(cred_path):
+        with open(cred_path, 'r', encoding='utf-8') as file:
+            for line in file:
+                if f'[{COREWEAVE_PROFILE_NAME}]' in line:
+                    coreweave_profile_exists = True
+                    break
+    return coreweave_profile_exists
+
+
+def get_credential_file_mounts() -> Dict[str, str]:
+    """Returns credential file mounts for CoreWeave.
+
+    Returns:
+        Dict[str, str]: A dictionary mapping source paths to destination paths
+        for credential files.
+    """
+    coreweave_credential_mounts = {
+        COREWEAVE_CREDENTIALS_PATH: COREWEAVE_CREDENTIALS_PATH,
+        COREWEAVE_CONFIG_PATH: COREWEAVE_CONFIG_PATH
+    }
+    return coreweave_credential_mounts

sky/adaptors/do.py

@@ -1,7 +1,5 @@
 """Digital Ocean cloud adaptors"""
 
-# pylint: disable=import-outside-toplevel
-
 from sky.adaptors import common
 
 _IMPORT_ERROR_MESSAGE = ('Failed to import dependencies for DO. '
@@ -16,5 +14,13 @@ _LAZY_MODULES = (pydo, azure)
 @common.load_lazy_modules(modules=_LAZY_MODULES)
 def exceptions():
     """Azure exceptions."""
+    # pylint: disable=import-outside-toplevel
     from azure.core import exceptions as azure_exceptions
     return azure_exceptions
+
+
+def check_exceptions_dependencies_installed():
+    """Check if the azure.core.exceptions module is installed."""
+    if not common.can_import_modules(['azure.core.exceptions']):
+        return False, _IMPORT_ERROR_MESSAGE
+    return True, None

sky/adaptors/hyperbolic.py

@@ -0,0 +1,8 @@
+"""Hyperbolic cloud adaptor."""
+
+from sky.adaptors import common
+
+hyperbolic = common.LazyImport(
+    'hyperbolic',
+    import_error_message='Failed to import dependencies for Hyperbolic. '
+    'Try running: pip install "skypilot[hyperbolic]"')