skypilot-nightly 1.0.0.dev20250509__py3-none-any.whl → 1.0.0.dev20251107__py3-none-any.whl

sky/setup_files/dependencies.py

@@ -8,11 +8,18 @@ This file is imported by setup.py, so:
 import sys
 from typing import Dict, List
 
+clouds_with_ray = ['ibm', 'docker', 'scp']
+
 install_requires = [
     'wheel<0.46.0',  # https://github.com/skypilot-org/skypilot/issues/5153
+    'setuptools',  # TODO: match version to pyproject.toml once #5153 is fixed
+    'pip',
     'cachetools',
     # NOTE: ray requires click>=7.0.
-    'click >= 7.0',
+    # click 8.2.0 has a bug in parsing the command line arguments:
+    # https://github.com/pallets/click/issues/2894
+    # TODO(aylei): remove this once the bug is fixed in click.
+    'click >= 7.0, < 8.2.0',
     'colorama',
     'cryptography',
     # Jinja has a bug in older versions because of the lack of pinning
@@ -32,7 +39,8 @@ install_requires = [
     # Light weight requirement, can be replaced with "typing" once
     # we deprecate Python 3.7 (this will take a while).
     'typing_extensions',
-    'filelock >= 3.6.0',
+    # filelock 3.15.0 or higher is required for async file locking.
+    'filelock >= 3.15.0',
     'packaging',
     'psutil',
     'pulp',
@@ -40,9 +48,18 @@ install_requires = [
     # (https://github.com/yaml/pyyaml/issues/601)
     # <= 3.13 may encounter https://github.com/ultralytics/yolov5/issues/414
     'pyyaml > 3.13, != 5.4.*',
+    'ijson',
+    'orjson',
     'requests',
+    # SkyPilot inherits from uvicorn.Server to customize the behavior of
+    # uvicorn, so we need to pin uvicorn version to avoid potential break
+    # changes.
+    # Notes for current version check:
+    # - uvicorn 0.33.0 is the latest version that supports Python 3.8
+    # - uvicorn 0.36.0 removes setup_event_loop thus breaks SkyPilot's custom
+    #   behavior.
+    'uvicorn[standard] >=0.33.0, <0.36.0',
     'fastapi',
-    'uvicorn[standard]',
     # Some pydantic versions are not compatible with ray. Adopted from ray's
     # setup.py:
     # https://github.com/ray-project/ray/blob/ray-2.9.3/python/setup.py#L254
@@ -53,27 +70,68 @@ install_requires = [
     'aiofiles',
     'httpx',
     'setproctitle',
+    'sqlalchemy',
+    'psycopg2-binary',
+    'aiosqlite',
+    'asyncpg',
+    # TODO(hailong): These three dependencies should be removed after we make
+    # the client-side actually not importing them.
+    'casbin',
+    'sqlalchemy_adapter',
+    # Required for API server metrics
+    'prometheus_client>=0.8.0',
+    'passlib',
+    'bcrypt==4.0.1',
+    'pyjwt',
+    'gitpython',
+    'types-paramiko',
+    'alembic',
+    'aiohttp',
+    'anyio',
+]
+
+# See requirements-dev.txt for the version of grpc and protobuf
+# used to generate the code during development.
+
+# The grpc version at runtime has to be newer than the version
+# used to generate the code.
+GRPC = 'grpcio>=1.63.0'
+# >= 5.26.1 because the runtime version can't be older than the version
+# used to generate the code.
+# < 7.0.0 because code generated for a major version V will be supported by
+# protobuf runtimes of version V and V+1.
+# https://protobuf.dev/support/cross-version-runtime-guarantee
+PROTOBUF = 'protobuf>=5.26.1, < 7.0.0'
+
+server_dependencies = [
+    # TODO: Some of these dependencies are also specified in install_requires,
+    # so they are redundant here. We should figure out if they are only needed
+    # on the server (should remove from install_requires), or if they are needed
+    # on the client (should remove from here).
+    'casbin',
+    'sqlalchemy_adapter',
+    'passlib',
+    'pyjwt',
+    'aiohttp',
+    'anyio',
+    GRPC,
+    PROTOBUF,
+    'aiosqlite',
+    'greenlet',
 ]
 
 local_ray = [
     # Lower version of ray will cause dependency conflict for
     # click/grpcio/protobuf.
-    # Excluded 2.6.0 as it has a bug in the cluster launcher:
+    # Ray 2.6.1+ resolved cluster launcher bugs
+    # and grpcio issues on Apple Silicon.
     # https://github.com/ray-project/ray/releases/tag/ray-2.6.1
-    'ray[default] >= 2.2.0, != 2.6.0',
+    'ray[default] >= 2.6.1',
 ]
 
 remote = [
-    # Adopted from ray's setup.py:
-    # https://github.com/ray-project/ray/blob/ray-2.9.3/python/setup.py#L251-L252
-    # SkyPilot: != 1.48.0 is required to avoid the error where ray dashboard
-    # fails to start when ray start is called (#2054).
-    # Tracking issue: https://github.com/ray-project/ray/issues/30984
-    'grpcio >= 1.32.0, != 1.48.0; python_version < \'3.10\'',
-    'grpcio >= 1.42.0, != 1.48.0; python_version >= \'3.10\'',
-    # Adopted from ray's setup.py:
-    # https://github.com/ray-project/ray/blob/ray-2.9.3/python/setup.py#L343
-    'protobuf >= 3.15.3, != 3.19.5',
+    GRPC,
+    PROTOBUF,
 ]
 
 # NOTE: Change the templates/jobs-controller.yaml.j2 file if any of the
@@ -90,11 +148,19 @@ aws_dependencies = [
     'colorama < 0.4.5',
 ]
 
+# Kubernetes 32.0.0 has an authentication bug:
+# https://github.com/kubernetes-client/python/issues/2333
+kubernetes_dependencies = [
+    'kubernetes>=20.0.0,!=32.0.0',
+    'websockets',
+    'python-dateutil',
+]
+
 # azure-cli cannot be installed normally by uv, so we need to work around it in
 # a few places.
 AZURE_CLI = 'azure-cli>=2.65.0'
 
-extras_require: Dict[str, List[str]] = {
+cloud_dependencies: Dict[str, List[str]] = {
     'aws': aws_dependencies,
     # TODO(zongheng): azure-cli is huge and takes a long time to install.
     # Tracked in: https://github.com/Azure/azure-cli/issues/7387
@@ -109,11 +175,18 @@ extras_require: Dict[str, List[str]] = {
         'azure-mgmt-compute>=33.0.0',
         'azure-storage-blob>=12.23.1',
         'msgraph-sdk',
-    ] + local_ray,
+        'msrestazure',
+    ],
     # We need google-api-python-client>=2.69.0 to enable 'discardLocalSsd'
     # parameter for stopping instances. Reference:
     # https://github.com/googleapis/google-api-python-client/commit/f6e9d3869ed605b06f7cbf2e8cf2db25108506e6
-    'gcp': ['google-api-python-client>=2.69.0', 'google-cloud-storage'],
+    'gcp': [
+        'google-api-python-client>=2.69.0',
+        'google-cloud-storage',
+        # see https://github.com/conda/conda/issues/13619
+        # see https://github.com/googleapis/google-api-python-client/issues/2554
+        'pyopenssl >= 23.2.0, <24.3.0',
+    ],
     'ibm': [
         'ibm-cloud-sdk-core',
         'ibm-vpc',
@@ -123,17 +196,20 @@ extras_require: Dict[str, List[str]] = {
     'docker': ['docker'] + local_ray,
     'lambda': [],  # No dependencies needed for lambda
     'cloudflare': aws_dependencies,
+    'coreweave': aws_dependencies + kubernetes_dependencies,
     'scp': local_ray,
-    'oci': ['oci'] + local_ray,
-    # Kubernetes 32.0.0 has an authentication bug: https://github.com/kubernetes-client/python/issues/2333 # pylint: disable=line-too-long
-    'kubernetes': ['kubernetes>=20.0.0,!=32.0.0', 'websockets'],
-    'remote': remote,
+    'oci': ['oci'],
+    'kubernetes': kubernetes_dependencies,
+    'ssh': kubernetes_dependencies,
     # For the container registry auth api. Reference:
     # https://github.com/runpod/runpod-python/releases/tag/1.6.1
-    'runpod': ['runpod>=1.6.1'],
+    # RunPod needs a TOML parser to read ~/.runpod/config.toml. On Python 3.11+
+    # stdlib provides tomllib; on lower versions we depend on tomli explicitly.
+    'runpod': ['runpod>=1.6.1', 'tomli; python_version < "3.11"'],
     'fluidstack': [],  # No dependencies needed for fluidstack
     'cudo': ['cudo-compute>=0.1.10'],
     'paperspace': [],  # No dependencies needed for paperspace
+    'primeintellect': [],  # No dependencies needed for primeintellect
     'do': ['pydo>=0.3.0', 'azure-core>=1.24.0', 'azure-common'],
     'vast': ['vastai-sdk>=0.1.12'],
     'vsphere': [
@@ -146,14 +222,43 @@ extras_require: Dict[str, List[str]] = {
         # 'vsphere-automation-sdk @ git+https://github.com/vmware/vsphere-automation-sdk-python.git@v8.0.1.0' pylint: disable=line-too-long
     ],
     'nebius': [
-        'nebius>=0.2.0',
-    ] + aws_dependencies
+        # Nebius requires grpcio and protobuf, so we need to include
+        # our constraints here.
+        'nebius>=0.2.47',
+        GRPC,
+        PROTOBUF,
+    ] + aws_dependencies,
+    'hyperbolic': [],  # No dependencies needed for hyperbolic
+    'seeweb': ['ecsapi>=0.2.0'],
+    'shadeform': [],  # No dependencies needed for shadeform
 }
 
-# Nebius needs python3.10. If python 3.9 [all] will not install nebius
+# Calculate which clouds should be included in the [all] installation.
+clouds_for_all = set(cloud_dependencies)
+
 if sys.version_info < (3, 10):
-    filtered_keys = [k for k in extras_require if k != 'nebius']
-    extras_require['all'] = sum(
-        [v for k, v in extras_require.items() if k != 'nebius'], [])
-else:
-    extras_require['all'] = sum(extras_require.values(), [])
+    # Nebius needs python3.10. If python 3.9 [all] will not install nebius
+    clouds_for_all.remove('nebius')
+    clouds_for_all.remove('seeweb')
+
+if sys.version_info >= (3, 12):
+    # The version of ray we use does not work with >= 3.12, so avoid clouds
+    # that require ray.
+    clouds_for_all -= set(clouds_with_ray)
+    # vast requires setuptools==51.1.1 which will not work with python >= 3.12
+    # TODO: Remove once https://github.com/vast-ai/vast-sdk/pull/6 is released
+    clouds_for_all.remove('vast')
+
+cloud_extras = {
+    cloud: dependencies + server_dependencies
+    for cloud, dependencies in cloud_dependencies.items()
+}
+
+extras_require: Dict[str, List[str]] = {
+    # Include server_dependencies with each cloud.
+    **cloud_extras,
+    'all': list(set().union(*[cloud_extras[cloud] for cloud in clouds_for_all])
+               ),
+    'remote': remote,
+    'server': server_dependencies,
+}

sky/setup_files/setup.py

@@ -148,45 +148,47 @@ if os.path.exists(readme_filepath):
     long_description = io.open(readme_filepath, 'r', encoding='utf-8').read()
     long_description = parse_readme(long_description)
 
-atexit.register(revert_commit_hash)
-replace_commit_hash()
-
-setuptools.setup(
-    # NOTE: this affects the package.whl wheel name. When changing this (if
-    # ever), you must grep for '.whl' and change all corresponding wheel paths
-    # (templates/*.j2 and wheel_utils.py).
-    name='skypilot-nightly',
-    version=find_version(),
-    packages=setuptools.find_packages(),
-    author='SkyPilot Team',
-    license='Apache 2.0',
-    readme='README.md',
-    description='SkyPilot: Run AI on Any Infra — Unified, Faster, Cheaper.',
-    long_description=long_description,
-    long_description_content_type='text/markdown',
-    setup_requires=['wheel'],
-    requires_python='>=3.7',
-    install_requires=dependencies['install_requires'],
-    extras_require=dependencies['extras_require'],
-    entry_points={
-        'console_scripts': ['sky = sky.cli:cli'],
-    },
-    include_package_data=True,
-    classifiers=[
-        'Programming Language :: Python :: 3.7',
-        'Programming Language :: Python :: 3.8',
-        'Programming Language :: Python :: 3.9',
-        'Programming Language :: Python :: 3.10',
-        'Programming Language :: Python :: 3.11',
-        'License :: OSI Approved :: Apache Software License',
-        'Operating System :: OS Independent',
-        'Topic :: Software Development :: Libraries :: Python Modules',
-        'Topic :: System :: Distributed Computing',
-    ],
-    project_urls={
-        'Homepage': 'https://github.com/skypilot-org/skypilot',
-        'Issues': 'https://github.com/skypilot-org/skypilot/issues',
-        'Discussion': 'https://github.com/skypilot-org/skypilot/discussions',
-        'Documentation': 'https://docs.skypilot.co/',
-    },
-)
+if __name__ == '__main__':
+    atexit.register(revert_commit_hash)
+    replace_commit_hash()
+    setuptools.setup(
+        # NOTE: this affects the package.whl wheel name. When changing this (if
+        # ever), you must grep for '.whl' and change all corresponding wheel paths
+        # (templates/*.j2 and wheel_utils.py).
+        name='skypilot-nightly',
+        version=find_version(),
+        packages=setuptools.find_packages(),
+        author='SkyPilot Team',
+        license='Apache 2.0',
+        readme='README.md',
+        description='SkyPilot: Run AI on Any Infra — Unified, Faster, Cheaper.',
+        long_description=long_description,
+        long_description_content_type='text/markdown',
+        setup_requires=['wheel'],
+        requires_python='>=3.7',
+        install_requires=dependencies['install_requires'],
+        extras_require=dependencies['extras_require'],
+        entry_points={
+            'console_scripts': ['sky = sky.cli:cli'],
+        },
+        include_package_data=True,
+        classifiers=[
+            'Programming Language :: Python :: 3.7',
+            'Programming Language :: Python :: 3.8',
+            'Programming Language :: Python :: 3.9',
+            'Programming Language :: Python :: 3.10',
+            'Programming Language :: Python :: 3.11',
+            'Programming Language :: Python :: 3.12',
+            'Programming Language :: Python :: 3.13',
+            'License :: OSI Approved :: Apache Software License',
+            'Operating System :: OS Independent',
+            'Topic :: Software Development :: Libraries :: Python Modules',
+            'Topic :: System :: Distributed Computing',
+        ],
+        project_urls={
+            'Homepage': 'https://github.com/skypilot-org/skypilot',
+            'Issues': 'https://github.com/skypilot-org/skypilot/issues',
+            'Discussion': 'https://github.com/skypilot-org/skypilot/discussions',
+            'Documentation': 'https://docs.skypilot.co/',
+        },
+    )

sky/sky_logging.py

@@ -10,6 +10,7 @@ import threading
 import colorama
 
 from sky.skylet import constants
+from sky.utils import context
 from sky.utils import env_options
 from sky.utils import rich_utils
 
@@ -18,6 +19,9 @@ _FORMAT = '%(levelname).1s %(asctime)s %(filename)s:%(lineno)d] %(message)s'
 _DATE_FORMAT = '%m-%d %H:%M:%S'
 _SENSITIVE_LOGGER = ['sky.provisioner', 'sky.optimizer']
 
+_DEBUG_LOG_DIR = os.path.expanduser(
+    os.path.join(constants.SKY_LOGS_DIRECTORY, 'request_debug'))
+
 DEBUG = logging.DEBUG
 INFO = logging.INFO
 WARNING = logging.WARNING
@@ -47,6 +51,43 @@ class NewLineFormatter(logging.Formatter):
         return msg
 
 
+class EnvAwareHandler(rich_utils.RichSafeStreamHandler):
+    """A handler that awares environment variables.
+
+    This handler dynamically reflects the log level from environment variables.
+    """
+
+    def __init__(self, stream=None, level=logging.NOTSET, sensitive=False):
+        super().__init__(stream)
+        self.level = level
+        self._sensitive = sensitive
+
+    @property
+    def level(self):
+        # Only refresh log level if we are in a context, since the log level
+        # has already been reloaded eagerly in multi-processing. Refresh again
+        # is a no-op and can be avoided.
+        # TODO(aylei): unify the mechanism for coroutine context and
+        # multi-processing.
+        if context.get() is not None:
+            if self._sensitive:
+                # For sensitive logger, suppress debug log despite the
+                # SKYPILOT_DEBUG env var if SUPPRESS_SENSITIVE_LOG is set
+                if env_options.Options.SUPPRESS_SENSITIVE_LOG.get():
+                    return logging.INFO
+            if env_options.Options.SHOW_DEBUG_INFO.get():
+                return logging.DEBUG
+            else:
+                return self._level
+        else:
+            return self._level
+
+    @level.setter
+    def level(self, level):
+        # pylint: disable=protected-access
+        self._level = logging._checkLevel(level)
+
+
 _root_logger = logging.getLogger('sky')
 _default_handler = None
 _logging_config = threading.local()
@@ -67,8 +108,7 @@ def _setup_logger():
     _root_logger.setLevel(logging.DEBUG)
     global _default_handler
     if _default_handler is None:
-        _default_handler = rich_utils.RichSafeStreamHandler(sys.stdout)
-        _default_handler.flush = sys.stdout.flush  # type: ignore
+        _default_handler = EnvAwareHandler(sys.stdout)
         if env_options.Options.SHOW_DEBUG_INFO.get():
             _default_handler.setLevel(logging.DEBUG)
         else:
@@ -87,8 +127,7 @@ def _setup_logger():
         # for certain loggers.
         for logger_name in _SENSITIVE_LOGGER:
             logger = logging.getLogger(logger_name)
-            handler_to_logger = rich_utils.RichSafeStreamHandler(sys.stdout)
-            handler_to_logger.flush = sys.stdout.flush  # type: ignore
+            handler_to_logger = EnvAwareHandler(sys.stdout, sensitive=True)
             logger.addHandler(handler_to_logger)
             logger.setLevel(logging.INFO)
             if _show_logging_prefix():
@@ -133,8 +172,41 @@ def set_logging_level(logger: str, level: int):
         logger.setLevel(original_level)
 
 
+@contextlib.contextmanager
+def set_sky_logging_levels(level: int):
+    """Set the logging level for all loggers."""
+    # Turn off logger
+    previous_levels = {}
+    for logger_name in logging.Logger.manager.loggerDict:
+        if logger_name.startswith('sky'):
+            logger = logging.getLogger(logger_name)
+            previous_levels[logger_name] = logger.level
+            logger.setLevel(level)
+    if level == logging.DEBUG:
+        previous_show_debug_info = env_options.Options.SHOW_DEBUG_INFO.get()
+        os.environ[env_options.Options.SHOW_DEBUG_INFO.env_key] = '1'
+    try:
+        yield
+    finally:
+        # Restore logger
+        for logger_name in logging.Logger.manager.loggerDict:
+            if logger_name.startswith('sky'):
+                logger = logging.getLogger(logger_name)
+                try:
+                    logger.setLevel(previous_levels[logger_name])
+                except KeyError:
+                    # New loggers maybe initialized after the context manager,
+                    # no need to restore the level.
+                    pass
+        if level == logging.DEBUG and not previous_show_debug_info:
+            os.environ.pop(env_options.Options.SHOW_DEBUG_INFO.env_key)
+
+
 def logging_enabled(logger: logging.Logger, level: int) -> bool:
-    return logger.level <= level
+    # Note(cooperc): This may return true in a lot of cases where we won't
+    # actually log anything, since the log level is set on the handler in
+    # _setup_logger.
+    return logger.getEffectiveLevel() <= level
 
 
 @contextlib.contextmanager
@@ -183,3 +255,28 @@ def generate_tmp_logging_file_path(file_name: str) -> str:
     log_path = os.path.expanduser(os.path.join(log_dir, file_name))
 
     return log_path
+
+
+@contextlib.contextmanager
+def add_debug_log_handler(request_id: str):
+    if os.getenv(constants.ENV_VAR_ENABLE_REQUEST_DEBUG_LOGGING) != 'true':
+        yield
+        return
+
+    os.makedirs(_DEBUG_LOG_DIR, exist_ok=True)
+    log_path = os.path.join(_DEBUG_LOG_DIR, f'{request_id}.log')
+    try:
+        debug_log_handler = logging.FileHandler(log_path)
+        debug_log_handler.setFormatter(FORMATTER)
+        debug_log_handler.setLevel(logging.DEBUG)
+        _root_logger.addHandler(debug_log_handler)
+        # sky.provision sets up its own logger/handler with propogate=False,
+        # so add it there too.
+        provision_logger = logging.getLogger('sky.provision')
+        provision_logger.addHandler(debug_log_handler)
+        provision_logger.setLevel(logging.DEBUG)
+        yield
+    finally:
+        _root_logger.removeHandler(debug_log_handler)
+        provision_logger.removeHandler(debug_log_handler)
+        debug_log_handler.close()

sky/skylet/attempt_skylet.py

@@ -12,6 +12,7 @@ def restart_skylet():
     # Kills old skylet if it is running.
     # TODO(zhwu): make the killing graceful, e.g., use a signal to tell
     # skylet to exit, instead of directly killing it.
+
     subprocess.run(
         # We use -m to grep instead of {constants.SKY_PYTHON_CMD} -m to grep
         # because need to handle the backward compatibility of the old skylet