skypilot-nightly 1.0.0.dev20250905__py3-none-any.whl → 1.0.0.dev20251210__py3-none-any.whl

sky/setup_files/dependencies.py

@@ -48,9 +48,18 @@ install_requires = [
     # (https://github.com/yaml/pyyaml/issues/601)
     # <= 3.13 may encounter https://github.com/ultralytics/yolov5/issues/414
     'pyyaml > 3.13, != 5.4.*',
+    'ijson',
+    'orjson',
     'requests',
+    # SkyPilot inherits from uvicorn.Server to customize the behavior of
+    # uvicorn, so we need to pin uvicorn version to avoid potential break
+    # changes.
+    # Notes for current version check:
+    # - uvicorn 0.33.0 is the latest version that supports Python 3.8
+    # - uvicorn 0.36.0 removes setup_event_loop thus breaks SkyPilot's custom
+    #   behavior.
+    'uvicorn[standard] >=0.33.0, <0.36.0',
     'fastapi',
-    'uvicorn[standard]',
     # Some pydantic versions are not compatible with ray. Adopted from ray's
     # setup.py:
     # https://github.com/ray-project/ray/blob/ray-2.9.3/python/setup.py#L254
@@ -63,6 +72,8 @@ install_requires = [
     'setproctitle',
     'sqlalchemy',
     'psycopg2-binary',
+    'aiosqlite',
+    'asyncpg',
     # TODO(hailong): These three dependencies should be removed after we make
     # the client-side actually not importing them.
     'casbin',
@@ -70,13 +81,13 @@ install_requires = [
     # Required for API server metrics
     'prometheus_client>=0.8.0',
     'passlib',
-    'bcrypt',
+    'bcrypt==4.0.1',
     'pyjwt',
     'gitpython',
+    'paramiko',
     'types-paramiko',
     'alembic',
     'aiohttp',
-    'aiosqlite',
     'anyio',
 ]
 
@@ -94,6 +105,10 @@ GRPC = 'grpcio>=1.63.0'
 PROTOBUF = 'protobuf>=5.26.1, < 7.0.0'
 
 server_dependencies = [
+    # TODO: Some of these dependencies are also specified in install_requires,
+    # so they are redundant here. We should figure out if they are only needed
+    # on the server (should remove from install_requires), or if they are needed
+    # on the client (should remove from here).
     'casbin',
     'sqlalchemy_adapter',
     'passlib',
@@ -103,14 +118,16 @@ server_dependencies = [
     GRPC,
     PROTOBUF,
     'aiosqlite',
+    'greenlet',
 ]
 
 local_ray = [
     # Lower version of ray will cause dependency conflict for
     # click/grpcio/protobuf.
-    # Excluded 2.6.0 as it has a bug in the cluster launcher:
+    # Ray 2.6.1+ resolved cluster launcher bugs
+    # and grpcio issues on Apple Silicon.
     # https://github.com/ray-project/ray/releases/tag/ray-2.6.1
-    'ray[default] >= 2.2.0, != 2.6.0',
+    'ray[default] >= 2.6.1',
 ]
 
 remote = [
@@ -132,11 +149,19 @@ aws_dependencies = [
     'colorama < 0.4.5',
 ]
 
+# Kubernetes 32.0.0 has an authentication bug:
+# https://github.com/kubernetes-client/python/issues/2333
+kubernetes_dependencies = [
+    'kubernetes>=20.0.0,!=32.0.0',
+    'websockets',
+    'python-dateutil',
+]
+
 # azure-cli cannot be installed normally by uv, so we need to work around it in
 # a few places.
 AZURE_CLI = 'azure-cli>=2.65.0'
 
-extras_require: Dict[str, List[str]] = {
+cloud_dependencies: Dict[str, List[str]] = {
     'aws': aws_dependencies,
     # TODO(zongheng): azure-cli is huge and takes a long time to install.
     # Tracked in: https://github.com/Azure/azure-cli/issues/7387
@@ -172,20 +197,23 @@ extras_require: Dict[str, List[str]] = {
     'docker': ['docker'] + local_ray,
     'lambda': [],  # No dependencies needed for lambda
     'cloudflare': aws_dependencies,
+    'coreweave': aws_dependencies + kubernetes_dependencies,
     'scp': local_ray,
     'oci': ['oci'],
-    # Kubernetes 32.0.0 has an authentication bug: https://github.com/kubernetes-client/python/issues/2333 # pylint: disable=line-too-long
-    'kubernetes': [
-        'kubernetes>=20.0.0,!=32.0.0', 'websockets', 'python-dateutil'
-    ],
-    'ssh': ['kubernetes>=20.0.0,!=32.0.0', 'websockets', 'python-dateutil'],
-    'remote': remote,
+    'kubernetes': kubernetes_dependencies,
+    'ssh': kubernetes_dependencies,
     # For the container registry auth api. Reference:
     # https://github.com/runpod/runpod-python/releases/tag/1.6.1
-    'runpod': ['runpod>=1.6.1'],
+    # RunPod needs a TOML parser to read ~/.runpod/config.toml. On Python 3.11+
+    # stdlib provides tomllib; on lower versions we depend on tomli explicitly.
+    # Instead of installing tomli conditionally, we install it explicitly.
+    # This is because the conditional installation of tomli does not work
+    # with controller package installation code.
+    'runpod': ['runpod>=1.6.1', 'tomli'],
     'fluidstack': [],  # No dependencies needed for fluidstack
     'cudo': ['cudo-compute>=0.1.10'],
     'paperspace': [],  # No dependencies needed for paperspace
+    'primeintellect': [],  # No dependencies needed for primeintellect
     'do': ['pydo>=0.3.0', 'azure-core>=1.24.0', 'azure-common'],
     'vast': ['vastai-sdk>=0.1.12'],
     'vsphere': [
@@ -198,19 +226,25 @@ extras_require: Dict[str, List[str]] = {
         # 'vsphere-automation-sdk @ git+https://github.com/vmware/vsphere-automation-sdk-python.git@v8.0.1.0' pylint: disable=line-too-long
     ],
     'nebius': [
-        'nebius>=0.2.47',
+        # Nebius requires grpcio and protobuf, so we need to include
+        # our constraints here.
+        'nebius>=0.3.12',
+        GRPC,
+        PROTOBUF,
     ] + aws_dependencies,
     'hyperbolic': [],  # No dependencies needed for hyperbolic
-    'server': server_dependencies,
+    'seeweb': ['ecsapi==0.4.0'],
+    'shadeform': [],  # No dependencies needed for shadeform
+    'slurm': [],  # No dependencies needed for slurm
 }
 
 # Calculate which clouds should be included in the [all] installation.
-clouds_for_all = set(extras_require)
-clouds_for_all.remove('remote')
+clouds_for_all = set(cloud_dependencies)
 
 if sys.version_info < (3, 10):
     # Nebius needs python3.10. If python 3.9 [all] will not install nebius
     clouds_for_all.remove('nebius')
+    clouds_for_all.remove('seeweb')
 
 if sys.version_info >= (3, 12):
     # The version of ray we use does not work with >= 3.12, so avoid clouds
@@ -220,5 +254,16 @@ if sys.version_info >= (3, 12):
     # TODO: Remove once https://github.com/vast-ai/vast-sdk/pull/6 is released
     clouds_for_all.remove('vast')
 
-extras_require['all'] = list(
-    set().union(*[extras_require[cloud] for cloud in clouds_for_all]))
+cloud_extras = {
+    cloud: dependencies + server_dependencies
+    for cloud, dependencies in cloud_dependencies.items()
+}
+
+extras_require: Dict[str, List[str]] = {
+    # Include server_dependencies with each cloud.
+    **cloud_extras,
+    'all': list(set().union(*[cloud_extras[cloud] for cloud in clouds_for_all])
+               ),
+    'remote': remote,
+    'server': server_dependencies,
+}

sky/setup_files/setup.py

@@ -148,47 +148,47 @@ if os.path.exists(readme_filepath):
     long_description = io.open(readme_filepath, 'r', encoding='utf-8').read()
     long_description = parse_readme(long_description)
 
-atexit.register(revert_commit_hash)
-replace_commit_hash()
-
-setuptools.setup(
-    # NOTE: this affects the package.whl wheel name. When changing this (if
-    # ever), you must grep for '.whl' and change all corresponding wheel paths
-    # (templates/*.j2 and wheel_utils.py).
-    name='skypilot-nightly',
-    version=find_version(),
-    packages=setuptools.find_packages(),
-    author='SkyPilot Team',
-    license='Apache 2.0',
-    readme='README.md',
-    description='SkyPilot: Run AI on Any Infra — Unified, Faster, Cheaper.',
-    long_description=long_description,
-    long_description_content_type='text/markdown',
-    setup_requires=['wheel'],
-    requires_python='>=3.7',
-    install_requires=dependencies['install_requires'],
-    extras_require=dependencies['extras_require'],
-    entry_points={
-        'console_scripts': ['sky = sky.cli:cli'],
-    },
-    include_package_data=True,
-    classifiers=[
-        'Programming Language :: Python :: 3.7',
-        'Programming Language :: Python :: 3.8',
-        'Programming Language :: Python :: 3.9',
-        'Programming Language :: Python :: 3.10',
-        'Programming Language :: Python :: 3.11',
-        'Programming Language :: Python :: 3.12',
-        'Programming Language :: Python :: 3.13',
-        'License :: OSI Approved :: Apache Software License',
-        'Operating System :: OS Independent',
-        'Topic :: Software Development :: Libraries :: Python Modules',
-        'Topic :: System :: Distributed Computing',
-    ],
-    project_urls={
-        'Homepage': 'https://github.com/skypilot-org/skypilot',
-        'Issues': 'https://github.com/skypilot-org/skypilot/issues',
-        'Discussion': 'https://github.com/skypilot-org/skypilot/discussions',
-        'Documentation': 'https://docs.skypilot.co/',
-    },
-)
+if __name__ == '__main__':
+    atexit.register(revert_commit_hash)
+    replace_commit_hash()
+    setuptools.setup(
+        # NOTE: this affects the package.whl wheel name. When changing this (if
+        # ever), you must grep for '.whl' and change all corresponding wheel paths
+        # (templates/*.j2 and wheel_utils.py).
+        name='skypilot-nightly',
+        version=find_version(),
+        packages=setuptools.find_packages(),
+        author='SkyPilot Team',
+        license='Apache 2.0',
+        readme='README.md',
+        description='SkyPilot: Run AI on Any Infra — Unified, Faster, Cheaper.',
+        long_description=long_description,
+        long_description_content_type='text/markdown',
+        setup_requires=['wheel'],
+        requires_python='>=3.7',
+        install_requires=dependencies['install_requires'],
+        extras_require=dependencies['extras_require'],
+        entry_points={
+            'console_scripts': ['sky = sky.cli:cli'],
+        },
+        include_package_data=True,
+        classifiers=[
+            'Programming Language :: Python :: 3.7',
+            'Programming Language :: Python :: 3.8',
+            'Programming Language :: Python :: 3.9',
+            'Programming Language :: Python :: 3.10',
+            'Programming Language :: Python :: 3.11',
+            'Programming Language :: Python :: 3.12',
+            'Programming Language :: Python :: 3.13',
+            'License :: OSI Approved :: Apache Software License',
+            'Operating System :: OS Independent',
+            'Topic :: Software Development :: Libraries :: Python Modules',
+            'Topic :: System :: Distributed Computing',
+        ],
+        project_urls={
+            'Homepage': 'https://github.com/skypilot-org/skypilot',
+            'Issues': 'https://github.com/skypilot-org/skypilot/issues',
+            'Discussion': 'https://github.com/skypilot-org/skypilot/discussions',
+            'Documentation': 'https://docs.skypilot.co/',
+        },
+    )

sky/sky_logging.py

@@ -85,7 +85,7 @@ class EnvAwareHandler(rich_utils.RichSafeStreamHandler):
     @level.setter
     def level(self, level):
         # pylint: disable=protected-access
-        self._level = logging._checkLevel(level)
+        self._level = logging._checkLevel(level)  # type: ignore[attr-defined]
 
 
 _root_logger = logging.getLogger('sky')
@@ -109,7 +109,6 @@ def _setup_logger():
     global _default_handler
     if _default_handler is None:
         _default_handler = EnvAwareHandler(sys.stdout)
-        _default_handler.flush = sys.stdout.flush  # type: ignore
         if env_options.Options.SHOW_DEBUG_INFO.get():
             _default_handler.setLevel(logging.DEBUG)
         else:
@@ -129,7 +128,6 @@ def _setup_logger():
         for logger_name in _SENSITIVE_LOGGER:
             logger = logging.getLogger(logger_name)
             handler_to_logger = EnvAwareHandler(sys.stdout, sensitive=True)
-            handler_to_logger.flush = sys.stdout.flush  # type: ignore
             logger.addHandler(handler_to_logger)
             logger.setLevel(logging.INFO)
             if _show_logging_prefix():
@@ -148,7 +146,8 @@ def reload_logger():
     such as SKYPILOT_DEBUG.
     """
     global _default_handler
-    _root_logger.removeHandler(_default_handler)
+    if _default_handler is not None:
+        _root_logger.removeHandler(_default_handler)
     _default_handler = None
     _setup_logger()
 
@@ -212,12 +211,21 @@ def logging_enabled(logger: logging.Logger, level: int) -> bool:
 
 
 @contextlib.contextmanager
-def silent():
+def silent(should_silence: bool = True):
     """Make all sky_logging.print() and logger.{info, warning...} silent.
 
     We preserve the ERROR level logging, so that errors are
     still printed.
+
+    Args:
+        should_silence: Whether to actually suppress the logging. If False, this
+            is a no-op context manager. Provided for convenience when we want to
+            suppress logging conditionally.
     """
+    if not should_silence:
+        yield
+        return
+
     global print
     previous_level = _root_logger.level
     previous_is_silent = is_silent()

sky/skylet/attempt_skylet.py

@@ -1,51 +1,143 @@
 """Restarts skylet if version does not match"""
 
 import os
+import signal
 import subprocess
+from typing import List, Optional, Tuple
+
+import psutil
 
 from sky.skylet import constants
+from sky.skylet import runtime_utils
+from sky.utils import common_utils
+
+VERSION_FILE = runtime_utils.get_runtime_dir_path(constants.SKYLET_VERSION_FILE)
+SKYLET_LOG_FILE = runtime_utils.get_runtime_dir_path(constants.SKYLET_LOG_FILE)
+PID_FILE = runtime_utils.get_runtime_dir_path(constants.SKYLET_PID_FILE)
+PORT_FILE = runtime_utils.get_runtime_dir_path(constants.SKYLET_PORT_FILE)
+
+
+def _is_running_skylet_process(pid: int) -> bool:
+    if pid <= 0:
+        return False
+    try:
+        process = psutil.Process(pid)
+        if not process.is_running():
+            return False
+        # Check if command line contains the skylet module identifier
+        cmdline = process.cmdline()
+        return any('sky.skylet.skylet' in arg for arg in cmdline)
+    except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess,
+            OSError) as e:
+        print(f'Error checking if skylet process {pid} is running: {e}')
+        return False
+
+
+def _find_running_skylet_pids() -> List[int]:
+    if os.path.exists(PID_FILE):
+        try:
+            with open(PID_FILE, 'r', encoding='utf-8') as pid_file:
+                pid = int(pid_file.read().strip())
+            if _is_running_skylet_process(pid):
+                return [pid]
+        except (OSError, ValueError, IOError) as e:
+            # Don't fallback to grep-based detection as the existence of the
+            # PID file implies that we are on the new version, and there is
+            # possibility of there being multiple skylet processes running,
+            # and we don't want to accidentally kill the wrong skylet(s).
+            print(f'Error reading PID file {PID_FILE}: {e}')
+        return []
+    else:
+        # Fall back to grep-based detection for backward compatibility.
+        pids = []
+        # We use -m to grep instead of {constants.SKY_PYTHON_CMD} -m to grep
+        # because need to handle the backward compatibility of the old skylet
+        # started before #3326, which does not use the full path to python.
+        proc = subprocess.run(
+            'ps aux | grep -v "grep" | grep "sky.skylet.skylet" | grep " -m"',
+            shell=True,
+            check=False,
+            capture_output=True,
+            text=True)
+        if proc.returncode == 0:
+            # Parse the output to extract PIDs (column 2)
+            for line in proc.stdout.strip().split('\n'):
+                if line:
+                    parts = line.split()
+                    if len(parts) >= 2:
+                        try:
+                            pids.append(int(parts[1]))
+                        except ValueError:
+                            continue
+        return pids
+
+
+def _check_version_match() -> Tuple[bool, Optional[str]]:
+    """Check if the version file matches the current skylet version.
 
-VERSION_FILE = os.path.expanduser(constants.SKYLET_VERSION_FILE)
+    Returns:
+        Tuple of (version_match: bool, version: str or None)
+    """
+    version: Optional[str] = None
+    if os.path.exists(VERSION_FILE):
+        try:
+            with open(VERSION_FILE, 'r', encoding='utf-8') as f:
+                version = f.read().strip()
+                return version == constants.SKYLET_VERSION, version
+        except (OSError, IOError):
+            pass
+    return False, version
 
 
 def restart_skylet():
     # Kills old skylet if it is running.
     # TODO(zhwu): make the killing graceful, e.g., use a signal to tell
     # skylet to exit, instead of directly killing it.
-    subprocess.run(
-        # We use -m to grep instead of {constants.SKY_PYTHON_CMD} -m to grep
-        # because need to handle the backward compatibility of the old skylet
-        # started before #3326, which does not use the full path to python.
-        'ps aux | grep "sky.skylet.skylet" | grep " -m "'
-        '| awk \'{print $2}\' | xargs kill >> ~/.sky/skylet.log 2>&1',
-        shell=True,
-        check=False)
+
+    # Find and kill running skylet processes
+    for pid in _find_running_skylet_pids():
+        try:
+            os.kill(pid, signal.SIGKILL)
+            # Wait until process fully terminates so its socket gets released.
+            # Without this, find_free_port may race with the kernel closing the
+            # socket and fail to bind to the port that's supposed to be free.
+            psutil.Process(pid).wait(timeout=5)
+        except (OSError, ProcessLookupError, psutil.NoSuchProcess,
+                psutil.TimeoutExpired):
+            # Process died between detection and kill, or timeout waiting
+            pass
+    # Clean up the PID file
+    try:
+        os.remove(PID_FILE)
+    except OSError:
+        pass  # Best effort cleanup
+
+    # TODO(kevin): Handle race conditions here. Race conditions can only
+    # happen on Slurm, where there could be multiple clusters running in
+    # one network namespace. For other clouds, the behaviour will be that
+    # it always gets port 46590 (default port).
+    port = common_utils.find_free_port(constants.SKYLET_GRPC_PORT)
     subprocess.run(
         # We have made sure that `attempt_skylet.py` is executed with the
         # skypilot runtime env activated, so that skylet can access the cloud
         # CLI tools.
-        f'nohup {constants.SKY_PYTHON_CMD} -m sky.skylet.skylet'
-        ' >> ~/.sky/skylet.log 2>&1 &',
+        f'nohup {constants.SKY_PYTHON_CMD} -m sky.skylet.skylet '
+        f'--port={port} '
+        f'>> {SKYLET_LOG_FILE} 2>&1 & echo $! > {PID_FILE}',
         shell=True,
         check=True)
+
+    with open(PORT_FILE, 'w', encoding='utf-8') as pf:
+        pf.write(str(port))
+
     with open(VERSION_FILE, 'w', encoding='utf-8') as v_f:
         v_f.write(constants.SKYLET_VERSION)
 
 
-proc = subprocess.run(
-    'ps aux | grep -v "grep" | grep "sky.skylet.skylet" | grep " -m"',
-    shell=True,
-    check=False)
-
-running = (proc.returncode == 0)
+# Check if our skylet is running
+running = bool(_find_running_skylet_pids())
 
-version_match = False
-found_version = None
-if os.path.exists(VERSION_FILE):
-    with open(VERSION_FILE, 'r', encoding='utf-8') as f:
-        found_version = f.read().strip()
-        if found_version == constants.SKYLET_VERSION:
-            version_match = True
+version_match, found_version = _check_version_match()
 
 version_string = (f' (found version {found_version}, new version '
                   f'{constants.SKYLET_VERSION})')

sky/skylet/configs.py

@@ -5,6 +5,7 @@ import pathlib
 import threading
 from typing import Callable, Optional, Union
 
+from sky.skylet import runtime_utils
 from sky.utils.db import db_utils
 
 _DB_PATH = None
@@ -29,7 +30,8 @@ def init_db(func: Callable):
 
         with _db_init_lock:
             if _DB_PATH is None:
-                _DB_PATH = os.path.expanduser('~/.sky/skylet_config.db')
+                _DB_PATH = runtime_utils.get_runtime_dir_path(
+                    '.sky/skylet_config.db')
                 os.makedirs(pathlib.Path(_DB_PATH).parents[0], exist_ok=True)
                 with db_utils.safe_cursor(
                         _DB_PATH