skypilot-nightly 1.0.0.dev20250827__py3-none-any.whl → 1.0.0.dev20250829__py3-none-any.whl

sky/skypilot_config.py

@@ -494,7 +494,7 @@ def reload_config() -> None:
 def parse_and_validate_config_file(config_path: str) -> config_utils.Config:
     config = config_utils.Config()
     try:
-        config_dict = common_utils.read_yaml(config_path)
+        config_dict = yaml_utils.read_yaml(config_path)
         config = config_utils.Config.from_dict(config_dict)
         # pop the db url from the config, and set it to the env var.
         # this is to avoid db url (considered a sensitive value)
@@ -504,7 +504,7 @@ def parse_and_validate_config_file(config_path: str) -> config_utils.Config:
             os.environ[constants.ENV_VAR_DB_CONNECTION_URI] = db_url
         if sky_logging.logging_enabled(logger, sky_logging.DEBUG):
             logger.debug(f'Config loaded from {config_path}:\n'
-                         f'{common_utils.dump_yaml_str(dict(config))}')
+                         f'{yaml_utils.dump_yaml_str(dict(config))}')
     except yaml.YAMLError as e:
         logger.error(f'Error in loading config file ({config_path}):', e)
     if config:
@@ -600,7 +600,7 @@ def _reload_config_as_server() -> None:
             sqlalchemy_engine.dispose()
     if sky_logging.logging_enabled(logger, sky_logging.DEBUG):
         logger.debug(f'server config: \n'
-                     f'{common_utils.dump_yaml_str(dict(server_config))}')
+                     f'{yaml_utils.dump_yaml_str(dict(server_config))}')
     _set_loaded_config(server_config)
     _set_loaded_config_path(server_config_path)
 
@@ -628,7 +628,7 @@ def _reload_config_as_client() -> None:
     if sky_logging.logging_enabled(logger, sky_logging.DEBUG):
         logger.debug(
             f'client config (before task and CLI overrides): \n'
-            f'{common_utils.dump_yaml_str(dict(overlaid_client_config))}')
+            f'{yaml_utils.dump_yaml_str(dict(overlaid_client_config))}')
     _set_loaded_config(overlaid_client_config)
     _set_loaded_config_path([user_config_path, project_config_path])
 
@@ -738,9 +738,9 @@ def override_skypilot_config(
                 'Failed to override the SkyPilot config on API '
                 'server with your local SkyPilot config:\n'
                 '=== SkyPilot config on API server ===\n'
-                f'{common_utils.dump_yaml_str(dict(original_config))}\n'
+                f'{yaml_utils.dump_yaml_str(dict(original_config))}\n'
                 '=== Your local SkyPilot config ===\n'
-                f'{common_utils.dump_yaml_str(dict(override_configs))}\n'
+                f'{yaml_utils.dump_yaml_str(dict(override_configs))}\n'
                 f'Details: {e}') from e
     finally:
         _set_loaded_config(original_config)
@@ -767,7 +767,7 @@ def replace_skypilot_config(new_configs: config_utils.Config) -> Iterator[None]:
                                          mode='w',
                                          prefix='mutated-skypilot-config-',
                                          suffix='.yaml') as temp_file:
-            common_utils.dump_yaml(temp_file.name, dict(**new_configs))
+            yaml_utils.dump_yaml(temp_file.name, dict(**new_configs))
         # Modify the env var of current process or context so that the
         # new config will be used by spawned sub-processes.
         # Note that this code modifies os.environ directly because it
@@ -831,7 +831,7 @@ def apply_cli_config(cli_config: Optional[List[str]]) -> Dict[str, Any]:
     parsed_config = _compose_cli_config(cli_config)
     if sky_logging.logging_enabled(logger, sky_logging.DEBUG):
         logger.debug(f'applying following CLI overrides: \n'
-                     f'{common_utils.dump_yaml_str(dict(parsed_config))}')
+                     f'{yaml_utils.dump_yaml_str(dict(parsed_config))}')
     _set_loaded_config(
         overlay_skypilot_config(original_config=_get_loaded_config(),
                                 override_configs=parsed_config))
@@ -875,7 +875,7 @@ def update_api_server_config_no_lock(config: config_utils.Config) -> None:
                 def _set_config_yaml_to_db(key: str,
                                            config: config_utils.Config):
                     assert sqlalchemy_engine is not None
-                    config_str = common_utils.dump_yaml_str(dict(config))
+                    config_str = yaml_utils.dump_yaml_str(dict(config))
                     with orm.Session(sqlalchemy_engine) as session:
                         if (sqlalchemy_engine.dialect.name ==
                                 db_utils.SQLAlchemyDialect.SQLITE.value):
@@ -901,7 +901,7 @@ def update_api_server_config_no_lock(config: config_utils.Config) -> None:
 
     if not db_updated:
         # save to the local file (PVC in Kubernetes, local file otherwise)
-        common_utils.dump_yaml(global_config_path, dict(config))
+        yaml_utils.dump_yaml(global_config_path, dict(config))
 
         if config_map_utils.is_running_in_kubernetes():
             # In Kubernetes, sync the PVC config to ConfigMap for user

sky/task.py

@@ -564,7 +564,7 @@ class Task:
         secrets_overrides: Optional[List[Tuple[str, str]]] = None,
     ) -> 'Task':
         user_specified_yaml = config.pop('_user_specified_yaml',
-                                         common_utils.dump_yaml_str(config))
+                                         yaml_utils.dump_yaml_str(config))
         # More robust handling for 'envs': explicitly convert keys and values to
         # str, since users may pass '123' as keys/values which will get parsed
         # as int causing validate_schema() to fail.

sky/templates/aws-ray.yml.j2

@@ -48,6 +48,7 @@ provider:
   # The upper-level SkyPilot code has make sure there will not be resource
   # leakage.
   disable_launch_config_check: true
+  max_efa_interfaces: {{max_efa_interfaces}}
 
 auth:
   ssh_user: {{ssh_user}}

sky/templates/nebius-ray.yml.j2

@@ -56,15 +56,11 @@ available_node_types:
           filesystem_mount_path: {{ fs.filesystem_mount_path }}
         {%- endfor %}
       UserData: |
-        runcmd:
-          - sudo sed -i 's/^#\?AllowTcpForwarding.*/AllowTcpForwarding yes/' /etc/ssh/sshd_config
-          - systemctl restart sshd
-
         {# Two available OS images:
-               1. ubuntu22.04-driverless - requires Docker installation
-               2. ubuntu22.04-cuda12 - comes with Docker pre-installed
-            To optimize deployment speed, Docker is only installed when using ubuntu22.04-driverless #}
-        {%- if docker_image is not none and image_id == 'ubuntu22.04-driverless' %}
+               1. ubuntu24.04-driverless - requires Docker installation
+               2. ubuntu24.04-cuda12 - comes with Docker pre-installed
+            To optimize deployment speed, Docker is only installed when using ubuntu24.04-driverless #}
+        {%- if docker_image is not none and image_id.endswith('-driverless') %}
         apt:
           sources:
             docker.list:

sky/usage/usage_lib.py

@@ -19,6 +19,7 @@ from sky.usage import constants
 from sky.utils import common_utils
 from sky.utils import env_options
 from sky.utils import ux_utils
+from sky.utils import yaml_utils
 
 if typing.TYPE_CHECKING:
     import inspect
@@ -402,7 +403,7 @@ def _clean_yaml(yaml_info: Dict[str, Optional[str]]):
                     contents = inspect.getsource(contents)
 
                 if type(contents) in constants.USAGE_MESSAGE_REDACT_TYPES:
-                    lines = common_utils.dump_yaml_str({
+                    lines = yaml_utils.dump_yaml_str({
                         redact_type: contents
                     }).strip().split('\n')
                     message = (f'{len(lines)} lines {redact_type.upper()}'
@@ -431,7 +432,7 @@ def prepare_json_from_yaml_config(
         with open(yaml_config_or_path, 'r', encoding='utf-8') as f:
             lines = f.readlines()
             comment_lines = [line for line in lines if line.startswith('#')]
-        yaml_info = common_utils.read_yaml_all(yaml_config_or_path)
+        yaml_info = yaml_utils.read_yaml_all(yaml_config_or_path)
 
     for i in range(len(yaml_info)):
         if yaml_info[i] is None:

sky/utils/annotations.py

@@ -7,7 +7,7 @@ from typing_extensions import ParamSpec
 
 # Whether the current process is a SkyPilot API server process.
 is_on_api_server = True
-FUNCTIONS_NEED_RELOAD_CACHE = []
+_FUNCTIONS_NEED_RELOAD_CACHE = []
 
 T = TypeVar('T')
 P = ParamSpec('P')
@@ -50,7 +50,13 @@ def lru_cache(scope: Literal['global', 'request'], *lru_cache_args,
         else:
             cached_func = functools.lru_cache(*lru_cache_args,
                                               **lru_cache_kwargs)(func)
-            FUNCTIONS_NEED_RELOAD_CACHE.append(cached_func)
+            _FUNCTIONS_NEED_RELOAD_CACHE.append(cached_func)
             return cached_func
 
     return decorator
+
+
+def clear_request_level_cache():
+    """Clear the request-level cache."""
+    for func in _FUNCTIONS_NEED_RELOAD_CACHE:
+        func.cache_clear()

sky/utils/cluster_utils.py

@@ -11,7 +11,7 @@ import uuid
 from sky.skylet import constants
 from sky.utils import command_runner
 from sky.utils import common_utils
-from sky.utils import timeline
+from sky.utils import lock_events
 
 # The cluster yaml used to create the current cluster where the module is
 # called.
@@ -107,7 +107,7 @@ class SSHConfigHelper(object):
         return auth_config['ssh_private_key']
 
     @classmethod
-    @timeline.FileLockEvent(ssh_conf_lock_path)
+    @lock_events.FileLockEvent(ssh_conf_lock_path)
     def add_cluster(
         cls,
         cluster_name: str,
@@ -334,7 +334,7 @@ class SSHConfigHelper(object):
             cluster_name: Cluster name.
         """
 
-        with timeline.FileLockEvent(
+        with lock_events.FileLockEvent(
                 cls.ssh_conf_per_cluster_lock_path.format(cluster_name)):
             cluster_config_path = os.path.expanduser(
                 cls.ssh_cluster_path.format(cluster_name))

sky/utils/common_utils.py

@@ -6,7 +6,6 @@ import functools
 import getpass
 import hashlib
 import inspect
-import io
 import os
 import platform
 import random
@@ -30,16 +29,13 @@ from sky.usage import constants as usage_constants
 from sky.utils import annotations
 from sky.utils import ux_utils
 from sky.utils import validator
-from sky.utils import yaml_utils
 
 if typing.TYPE_CHECKING:
     import jinja2
     import psutil
-    import yaml
 else:
     jinja2 = adaptors_common.LazyImport('jinja2')
     psutil = adaptors_common.LazyImport('psutil')
-    yaml = adaptors_common.LazyImport('yaml')
 
 USER_HASH_FILE = os.path.expanduser('~/.sky/user_hash')
 USER_HASH_LENGTH = 8
@@ -574,74 +570,6 @@ def user_and_hostname_hash() -> str:
     return f'{getpass.getuser()}-{hostname_hash}'
 
 
-def read_yaml(path: Optional[str]) -> Dict[str, Any]:
-    if path is None:
-        raise ValueError('Attempted to read a None YAML.')
-    with open(path, 'r', encoding='utf-8') as f:
-        config = yaml_utils.safe_load(f)
-    return config
-
-
-def read_yaml_all_str(yaml_str: str) -> List[Dict[str, Any]]:
-    stream = io.StringIO(yaml_str)
-    config = yaml_utils.safe_load_all(stream)
-    configs = list(config)
-    if not configs:
-        # Empty YAML file.
-        return [{}]
-    return configs
-
-
-def read_yaml_all(path: str) -> List[Dict[str, Any]]:
-    with open(path, 'r', encoding='utf-8') as f:
-        return read_yaml_all_str(f.read())
-
-
-def dump_yaml(path: str,
-              config: Union[List[Dict[str, Any]], Dict[str, Any]],
-              blank: bool = False) -> None:
-    """Dumps a YAML file.
-
-    Args:
-        path: the path to the YAML file.
-        config: the configuration to dump.
-    """
-    with open(path, 'w', encoding='utf-8') as f:
-        contents = dump_yaml_str(config)
-        if blank and isinstance(config, dict) and len(config) == 0:
-            # when dumping to yaml, an empty dict will go in as {}.
-            contents = ''
-        f.write(contents)
-
-
-def dump_yaml_str(config: Union[List[Dict[str, Any]], Dict[str, Any]]) -> str:
-    """Dumps a YAML string.
-
-    Args:
-        config: the configuration to dump.
-
-    Returns:
-        The YAML string.
-    """
-
-    # https://github.com/yaml/pyyaml/issues/127
-    class LineBreakDumper(yaml.SafeDumper):
-
-        def write_line_break(self, data=None):
-            super().write_line_break(data)
-            if len(self.indents) == 1:
-                super().write_line_break()
-
-    if isinstance(config, list):
-        dump_func = yaml.dump_all  # type: ignore
-    else:
-        dump_func = yaml.dump  # type: ignore
-    return dump_func(config,
-                     Dumper=LineBreakDumper,
-                     sort_keys=False,
-                     default_flow_style=False)
-
-
 def make_decorator(cls, name_or_fn: Union[str, Callable],
                    **ctx_kwargs) -> Callable:
     """Make the cls a decorator.

sky/utils/controller_utils.py

@@ -38,6 +38,7 @@ from sky.utils import env_options
 from sky.utils import registry
 from sky.utils import rich_utils
 from sky.utils import ux_utils
+from sky.utils import yaml_utils
 
 if typing.TYPE_CHECKING:
     import psutil
@@ -497,7 +498,7 @@ def shared_controller_vars_to_fill(
         with tempfile.NamedTemporaryFile(
                 delete=False,
                 suffix=_LOCAL_SKYPILOT_CONFIG_PATH_SUFFIX) as temp_file:
-            common_utils.dump_yaml(temp_file.name, dict(**local_user_config))
+            yaml_utils.dump_yaml(temp_file.name, dict(**local_user_config))
         local_user_config_path = temp_file.name
 
     vars_to_fill: Dict[str, Any] = {
@@ -786,9 +787,9 @@ def replace_skypilot_config_path_in_file_mounts(
             continue
         if local_path.endswith(_LOCAL_SKYPILOT_CONFIG_PATH_SUFFIX):
             with tempfile.NamedTemporaryFile('w', delete=False) as f:
-                user_config = common_utils.read_yaml(local_path)
+                user_config = yaml_utils.read_yaml(local_path)
                 config = _setup_proxy_command_on_controller(cloud, user_config)
-                common_utils.dump_yaml(f.name, dict(**config))
+                yaml_utils.dump_yaml(f.name, dict(**config))
                 file_mounts[remote_path] = f.name
                 replaced = True
     if replaced:

sky/utils/dag_utils.py

@@ -6,9 +6,9 @@ from sky import dag as dag_lib
 from sky import sky_logging
 from sky import task as task_lib
 from sky.utils import cluster_utils
-from sky.utils import common_utils
 from sky.utils import registry
 from sky.utils import ux_utils
+from sky.utils import yaml_utils
 
 logger = sky_logging.init_logger(__name__)
 
@@ -117,7 +117,7 @@ def load_chain_dag_from_yaml(
       A chain Dag with 1 or more tasks (an empty entrypoint would create a
       trivial task).
     """
-    configs = common_utils.read_yaml_all(path)
+    configs = yaml_utils.read_yaml_all(path)
     return _load_chain_dag(configs, env_overrides, secret_overrides)
 
 
@@ -143,7 +143,7 @@ def load_chain_dag_from_yaml_str(
       A chain Dag with 1 or more tasks (an empty entrypoint would create a
       trivial task).
     """
-    configs = common_utils.read_yaml_all_str(yaml_str)
+    configs = yaml_utils.read_yaml_all_str(yaml_str)
     return _load_chain_dag(configs, env_overrides, secrets_overrides)
 
 
@@ -164,7 +164,7 @@ def dump_chain_dag_to_yaml_str(dag: dag_lib.Dag,
         configs.append(
             task.to_yaml_config(
                 use_user_specified_yaml=use_user_specified_yaml))
-    return common_utils.dump_yaml_str(configs)
+    return yaml_utils.dump_yaml_str(configs)
 
 
 def dump_chain_dag_to_yaml(dag: dag_lib.Dag, path: str) -> None:

sky/utils/db/db_utils.py

@@ -1,4 +1,5 @@
 """Utils for sky databases."""
+import asyncio
 import contextlib
 import enum
 import sqlite3
@@ -6,6 +7,7 @@ import threading
 import typing
 from typing import Any, Callable, Optional
 
+import aiosqlite
 import sqlalchemy
 from sqlalchemy import exc as sqlalchemy_exc
 
@@ -283,3 +285,12 @@ class SQLiteConn(threading.local):
         self.conn = sqlite3.connect(db_path, timeout=_DB_TIMEOUT_S)
         self.cursor = self.conn.cursor()
         create_table(self.cursor, self.conn)
+        self._async_conn: Optional[aiosqlite.Connection] = None
+        self._async_conn_lock = asyncio.Lock()
+
+    async def async_conn(self) -> aiosqlite.Connection:
+        if self._async_conn is None:
+            async with self._async_conn_lock:
+                if self._async_conn is None:
+                    self._async_conn = await aiosqlite.connect(self.db_path)
+        return self._async_conn

sky/utils/db/migration_utils.py

@@ -19,7 +19,7 @@ logger = sky_logging.init_logger(__name__)
 DB_INIT_LOCK_TIMEOUT_SECONDS = 10
 
 GLOBAL_USER_STATE_DB_NAME = 'state_db'
-GLOBAL_USER_STATE_VERSION = '006'
+GLOBAL_USER_STATE_VERSION = '007'
 GLOBAL_USER_STATE_LOCK_PATH = '~/.sky/locks/.state_db.lock'
 
 SPOT_JOBS_DB_NAME = 'spot_jobs_db'

sky/utils/kubernetes/config_map_utils.py

@@ -4,7 +4,7 @@ import os
 from sky import sky_logging
 from sky import skypilot_config
 from sky.adaptors import kubernetes
-from sky.utils import common_utils
+from sky.utils import yaml_utils
 
 logger = sky_logging.init_logger(__name__)
 
@@ -69,7 +69,7 @@ def initialize_configmap_sync_on_startup(config_file_path: str) -> None:
 
         current_config = skypilot_config.parse_and_validate_config_file(
             config_file_path)
-        config_yaml = common_utils.dump_yaml_str(dict(current_config))
+        config_yaml = yaml_utils.dump_yaml_str(dict(current_config))
 
         configmap_body = {
             'apiVersion': 'v1',
@@ -111,7 +111,7 @@ def patch_configmap_with_config(config, config_file_path: str) -> None:
     try:
         namespace = _get_kubernetes_namespace()
         configmap_name = _get_configmap_name()
-        config_yaml = common_utils.dump_yaml_str(dict(config))
+        config_yaml = yaml_utils.dump_yaml_str(dict(config))
         patch_body = {'data': {'config.yaml': config_yaml}}
 
         try:

sky/utils/kubernetes_enums.py

@@ -42,4 +42,5 @@ class KubernetesAutoscalerType(enum.Enum):
     """Enum for the different types of cluster autoscalers for Kubernetes."""
     GKE = 'gke'
     KARPENTER = 'karpenter'
+    COREWEAVE = 'coreweave'
     GENERIC = 'generic'

sky/utils/lock_events.py

@@ -0,0 +1,94 @@
+"""Lock events."""
+
+import functools
+import os
+from typing import Optional, Union
+
+import filelock
+
+from sky.utils import locks
+from sky.utils import timeline
+
+
+class DistributedLockEvent:
+    """Serve both as a distributed lock and event for the lock."""
+
+    def __init__(self, lock_id: str, timeout: Optional[float] = None):
+        self._lock_id = lock_id
+        self._lock = locks.get_lock(lock_id, timeout)
+        self._hold_lock_event = timeline.Event(
+            f'[DistributedLock.hold]:{lock_id}')
+
+    def acquire(self):
+        was_locked = self._lock.is_locked
+        with timeline.Event(f'[DistributedLock.acquire]:{self._lock_id}'):
+            self._lock.acquire()
+        if not was_locked and self._lock.is_locked:
+            # start holding the lock after initial acquiring
+            self._hold_lock_event.begin()
+
+    def release(self):
+        was_locked = self._lock.is_locked
+        self._lock.release()
+        if was_locked and not self._lock.is_locked:
+            # stop holding the lock after initial releasing
+            self._hold_lock_event.end()
+
+    def __enter__(self):
+        self.acquire()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.release()
+
+    def __call__(self, f):
+
+        @functools.wraps(f)
+        def wrapper(*args, **kwargs):
+            with self:
+                return f(*args, **kwargs)
+
+        return wrapper
+
+
+class FileLockEvent:
+    """Serve both as a file lock and event for the lock."""
+
+    def __init__(self, lockfile: Union[str, os.PathLike], timeout: float = -1):
+        self._lockfile = lockfile
+        os.makedirs(os.path.dirname(os.path.abspath(self._lockfile)),
+                    exist_ok=True)
+        self._lock = filelock.FileLock(self._lockfile, timeout)
+        self._hold_lock_event = timeline.Event(
+            f'[FileLock.hold]:{self._lockfile}')
+
+    def acquire(self):
+        was_locked = self._lock.is_locked
+        with timeline.Event(f'[FileLock.acquire]:{self._lockfile}'):
+            self._lock.acquire()
+        if not was_locked and self._lock.is_locked:
+            # start holding the lock after initial acquiring
+            self._hold_lock_event.begin()
+
+    def release(self):
+        was_locked = self._lock.is_locked
+        self._lock.release()
+        if was_locked and not self._lock.is_locked:
+            # stop holding the lock after initial releasing
+            self._hold_lock_event.end()
+
+    def __enter__(self):
+        self.acquire()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.release()
+
+    def __call__(self, f):
+        # Make this class callable as a decorator.
+        @functools.wraps(f)
+        def wrapper(*args, **kwargs):
+            with self:
+                return f(*args, **kwargs)
+
+        return wrapper

sky/utils/schemas.py

@@ -1555,6 +1555,9 @@ def get_config_schema():
             'cluster_event_retention_hours': {
                 'type': 'number',
             },
+            'cluster_debug_event_retention_hours': {
+                'type': 'number',
+            },
         }
     }