skypilot-nightly 1.0.0.dev20240923__py3-none-any.whl → 1.0.0.dev20240925__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = 'd602225c897f10ca67a2d5b5db21982c0dc8c1ec'
+_SKYPILOT_COMMIT_SHA = 'e13c39104cc3fca974e2afa207bcca24817f4e17'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20240923'
+__version__ = '1.0.0.dev20240925'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 
@@ -82,6 +82,9 @@ _set_http_proxy_env_vars()
 from sky import backends
 from sky import benchmark
 from sky import clouds
+from sky.admin_policy import AdminPolicy
+from sky.admin_policy import MutatedUserRequest
+from sky.admin_policy import UserRequest
 from sky.clouds.service_catalog import list_accelerators
 from sky.core import autostop
 from sky.core import cancel
@@ -112,6 +115,7 @@ from sky.optimizer import Optimizer
 from sky.optimizer import OptimizeTarget
 from sky.resources import Resources
 from sky.skylet.job_lib import JobStatus
+from sky.skypilot_config import Config
 from sky.status_lib import ClusterStatus
 from sky.task import Task
 
@@ -185,4 +189,9 @@ __all__ = [
     # core APIs Storage Management
     'storage_ls',
     'storage_delete',
+    # Admin Policy
+    'UserRequest',
+    'MutatedUserRequest',
+    'AdminPolicy',
+    'Config',
 ]

sky/admin_policy.py

@@ -0,0 +1,101 @@
+"""Interface for admin-defined policy for user requests."""
+import abc
+import dataclasses
+import typing
+from typing import Optional
+
+if typing.TYPE_CHECKING:
+    import sky
+
+
+@dataclasses.dataclass
+class RequestOptions:
+    """Request options for admin policy.
+
+    Args:
+        cluster_name: Name of the cluster to create/reuse. It is None if not
+            specified by the user.
+        idle_minutes_to_autostop: Autostop setting requested by a user. The
+            cluster will be set to autostop after this many minutes of idleness.
+        down: If true, use autodown rather than autostop.
+        dryrun: Is the request a dryrun?
+    """
+    cluster_name: Optional[str]
+    idle_minutes_to_autostop: Optional[int]
+    down: bool
+    dryrun: bool
+
+
+@dataclasses.dataclass
+class UserRequest:
+    """A user request.
+
+    A "user request" is defined as a `sky launch / exec` command or its API
+    equivalent.
+
+    `sky jobs launch / serve up` involves multiple launch requests, including
+    the launch of controller and clusters for a job (which can have multiple
+    tasks if it is a pipeline) or service replicas. Each launch is a separate
+    request.
+
+    This class wraps the underlying task, the global skypilot config used to run
+    a task, and the request options.
+
+    Args:
+        task: User specified task.
+        skypilot_config: Global skypilot config to be used in this request.
+        request_options: Request options. It is None for jobs and services.
+    """
+    task: 'sky.Task'
+    skypilot_config: 'sky.Config'
+    request_options: Optional['RequestOptions'] = None
+
+
+@dataclasses.dataclass
+class MutatedUserRequest:
+    task: 'sky.Task'
+    skypilot_config: 'sky.Config'
+
+
+# pylint: disable=line-too-long
+class AdminPolicy:
+    """Abstract interface of an admin-defined policy for all user requests.
+
+    Admins can implement a subclass of AdminPolicy with the following signature:
+
+        import sky
+
+        class SkyPilotPolicyV1(sky.AdminPolicy):
+            def validate_and_mutate(user_request: UserRequest) -> MutatedUserRequest:
+                ...
+                return MutatedUserRequest(task=..., skypilot_config=...)
+
+    The policy can mutate both task and skypilot_config. Admins then distribute
+    a simple module that contains this implementation, installable in a way
+    that it can be imported by users from the same Python environment where
+    SkyPilot is running.
+
+    Users can register a subclass of AdminPolicy in the SkyPilot config file
+    under the key 'admin_policy', e.g.
+
+        admin_policy: my_package.SkyPilotPolicyV1
+    """
+
+    @classmethod
+    @abc.abstractmethod
+    def validate_and_mutate(cls,
+                            user_request: UserRequest) -> MutatedUserRequest:
+        """Validates and mutates the user request and returns mutated request.
+
+        Args:
+            user_request: The user request to validate and mutate.
+                UserRequest contains (sky.Task, sky.Config)
+
+        Returns:
+            MutatedUserRequest: The mutated user request.
+
+        Raises:
+            Exception to throw if the user request failed the validation.
+        """
+        raise NotImplementedError(
+            'Your policy must implement validate_and_mutate')

sky/backends/cloud_vm_ray_backend.py

@@ -4147,11 +4147,21 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
                     idle_minutes_to_autostop >= 0):
                 # We should hit this code path only for the controllers on
                 # Kubernetes and RunPod clusters.
-                assert (controller_utils.Controllers.from_name(
-                    handle.cluster_name) is not None), handle.cluster_name
-                logger.info('Auto-stop is not supported for Kubernetes '
-                            'and RunPod clusters. Skipping.')
-                return
+                controller = controller_utils.Controllers.from_name(
+                    handle.cluster_name)
+                assert (controller is not None), handle.cluster_name
+                if (controller
+                        == controller_utils.Controllers.SKY_SERVE_CONTROLLER and
+                        isinstance(handle.launched_resources.cloud,
+                                   clouds.Kubernetes)):
+                    # For SkyServe controllers on Kubernetes: override autostop
+                    # behavior to force autodown (instead of no-op)
+                    # to avoid dangling controllers.
+                    down = True
+                else:
+                    logger.info('Auto-stop is not supported for Kubernetes '
+                                'and RunPod clusters. Skipping.')
+                    return
 
             # Check if we're stopping spot
             assert (handle.launched_resources is not None and

sky/dag.py

@@ -1,8 +1,12 @@
 """DAGs: user applications to be run."""
 import pprint
 import threading
+import typing
 from typing import List, Optional
 
+if typing.TYPE_CHECKING:
+    from sky import task
+
 
 class Dag:
     """Dag: a user application, represented as a DAG of Tasks.
@@ -13,37 +17,37 @@ class Dag:
         >>>     task = sky.Task(...)
     """
 
-    def __init__(self):
-        self.tasks = []
+    def __init__(self) -> None:
+        self.tasks: List['task.Task'] = []
         import networkx as nx  # pylint: disable=import-outside-toplevel
 
         self.graph = nx.DiGraph()
-        self.name = None
+        self.name: Optional[str] = None
 
-    def add(self, task):
+    def add(self, task: 'task.Task') -> None:
         self.graph.add_node(task)
         self.tasks.append(task)
 
-    def remove(self, task):
+    def remove(self, task: 'task.Task') -> None:
         self.tasks.remove(task)
         self.graph.remove_node(task)
 
-    def add_edge(self, op1, op2):
+    def add_edge(self, op1: 'task.Task', op2: 'task.Task') -> None:
         assert op1 in self.graph.nodes
         assert op2 in self.graph.nodes
         self.graph.add_edge(op1, op2)
 
-    def __len__(self):
+    def __len__(self) -> int:
         return len(self.tasks)
 
-    def __enter__(self):
+    def __enter__(self) -> 'Dag':
         push_dag(self)
         return self
 
-    def __exit__(self, exc_type, exc_value, traceback):
+    def __exit__(self, exc_type, exc_value, traceback) -> None:
         pop_dag()
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         pformat = pprint.pformat(self.tasks)
         return f'DAG:\n{pformat}'
 
@@ -70,15 +74,15 @@ class Dag:
 
 class _DagContext(threading.local):
     """A thread-local stack of Dags."""
-    _current_dag = None
+    _current_dag: Optional[Dag] = None
     _previous_dags: List[Dag] = []
 
-    def push_dag(self, dag):
+    def push_dag(self, dag: Dag):
         if self._current_dag is not None:
             self._previous_dags.append(self._current_dag)
         self._current_dag = dag
 
-    def pop_dag(self):
+    def pop_dag(self) -> Optional[Dag]:
         old_dag = self._current_dag
         if self._previous_dags:
             self._current_dag = self._previous_dags.pop()

sky/exceptions.py

@@ -286,3 +286,8 @@ class ServeUserTerminatedError(Exception):
 
 class PortDoesNotExistError(Exception):
     """Raised when the port does not exist."""
+
+
+class UserRequestRejectedByPolicy(Exception):
+    """Raised when a user request is rejected by an admin policy."""
+    pass

sky/execution.py

@@ -9,6 +9,7 @@ from typing import List, Optional, Tuple, Union
 import colorama
 
 import sky
+from sky import admin_policy
 from sky import backends
 from sky import clouds
 from sky import global_user_state
@@ -16,6 +17,7 @@ from sky import optimizer
 from sky import sky_logging
 from sky.backends import backend_utils
 from sky.usage import usage_lib
+from sky.utils import admin_policy_utils
 from sky.utils import controller_utils
 from sky.utils import dag_utils
 from sky.utils import env_options
@@ -158,7 +160,16 @@ def _execute(
       handle: Optional[backends.ResourceHandle]; the handle to the cluster. None
         if dryrun.
     """
+
     dag = dag_utils.convert_entrypoint_to_dag(entrypoint)
+    dag, _ = admin_policy_utils.apply(
+        dag,
+        request_options=admin_policy.RequestOptions(
+            cluster_name=cluster_name,
+            idle_minutes_to_autostop=idle_minutes_to_autostop,
+            down=down,
+            dryrun=dryrun,
+        ))
     assert len(dag) == 1, f'We support 1 task for now. {dag}'
     task = dag.tasks[0]
 
@@ -170,9 +181,8 @@ def _execute(
 
     cluster_exists = False
     if cluster_name is not None:
-        existing_handle = global_user_state.get_handle_from_cluster_name(
-            cluster_name)
-        cluster_exists = existing_handle is not None
+        cluster_record = global_user_state.get_cluster_from_name(cluster_name)
+        cluster_exists = cluster_record is not None
         # TODO(woosuk): If the cluster exists, print a warning that
         # `cpus` and `memory` are not used as a job scheduling constraint,
         # unlike `gpus`.

sky/jobs/controller.py

@@ -64,6 +64,7 @@ class JobsController:
             if len(self._dag.tasks) <= 1:
                 task_name = self._dag_name
             else:
+                assert task.name is not None, task
                 task_name = task.name
                 # This is guaranteed by the spot_launch API, where we fill in
                 # the task.name with
@@ -447,6 +448,7 @@ def _cleanup(job_id: int, dag_yaml: str):
     # controller, we should keep it in sync with JobsController.__init__()
     dag, _ = _get_dag_and_name(dag_yaml)
     for task in dag.tasks:
+        assert task.name is not None, task
         cluster_name = managed_job_utils.generate_managed_job_cluster_name(
             task.name, job_id)
         recovery_strategy.terminate_cluster(cluster_name)

sky/jobs/core.py

@@ -18,6 +18,7 @@ from sky.jobs import constants as managed_job_constants
 from sky.jobs import utils as managed_job_utils
 from sky.skylet import constants as skylet_constants
 from sky.usage import usage_lib
+from sky.utils import admin_policy_utils
 from sky.utils import common_utils
 from sky.utils import controller_utils
 from sky.utils import dag_utils
@@ -54,6 +55,8 @@ def launch(
     dag_uuid = str(uuid.uuid4().hex[:4])
 
     dag = dag_utils.convert_entrypoint_to_dag(entrypoint)
+    dag, mutated_user_config = admin_policy_utils.apply(
+        dag, use_mutated_config_in_current_request=False)
     if not dag.is_chain():
         with ux_utils.print_exception_no_traceback():
             raise ValueError('Only single-task or chain DAG is '
@@ -103,6 +106,7 @@ def launch(
             **controller_utils.shared_controller_vars_to_fill(
                 controller_utils.Controllers.JOBS_CONTROLLER,
                 remote_user_config_path=remote_user_config_path,
+                local_user_config=mutated_user_config,
             ),
         }
 

sky/serve/core.py

@@ -17,6 +17,7 @@ from sky.serve import serve_state
 from sky.serve import serve_utils
 from sky.skylet import constants
 from sky.usage import usage_lib
+from sky.utils import admin_policy_utils
 from sky.utils import common_utils
 from sky.utils import controller_utils
 from sky.utils import resources_utils
@@ -124,6 +125,10 @@ def up(
 
     _validate_service_task(task)
 
+    dag, mutated_user_config = admin_policy_utils.apply(
+        task, use_mutated_config_in_current_request=False)
+    task = dag.tasks[0]
+
     controller_utils.maybe_translate_local_file_mounts_and_sync_up(task,
                                                                    path='serve')
 
@@ -158,6 +163,7 @@ def up(
             **controller_utils.shared_controller_vars_to_fill(
                 controller=controller_utils.Controllers.SKY_SERVE_CONTROLLER,
                 remote_user_config_path=remote_config_yaml_path,
+                local_user_config=mutated_user_config,
             ),
         }
         common_utils.fill_template(serve_constants.CONTROLLER_TEMPLATE,

sky/skypilot_config.py

@@ -61,6 +61,8 @@ from sky.utils import common_utils
 from sky.utils import schemas
 from sky.utils import ux_utils
 
+logger = sky_logging.init_logger(__name__)
+
 # The config path is discovered in this order:
 #
 # (1) (Used internally) If env var {ENV_VAR_SKYPILOT_CONFIG} exists, use its
@@ -78,11 +80,57 @@ ENV_VAR_SKYPILOT_CONFIG = 'SKYPILOT_CONFIG'
 # Path to the local config file.
 CONFIG_PATH = '~/.sky/config.yaml'
 
-logger = sky_logging.init_logger(__name__)
+
+class Config(Dict[str, Any]):
+    """SkyPilot config that supports setting/getting values with nested keys."""
+
+    def get_nested(self,
+                   keys: Tuple[str, ...],
+                   default_value: Any,
+                   override_configs: Optional[Dict[str, Any]] = None) -> Any:
+        """Gets a nested key.
+
+        If any key is not found, or any intermediate key does not point to a
+        dict value, returns 'default_value'.
+
+        Args:
+            keys: A tuple of strings representing the nested keys.
+            default_value: The default value to return if the key is not found.
+            override_configs: A dict of override configs with the same schema as
+                the config file, but only containing the keys to override.
+
+        Returns:
+            The value of the nested key, or 'default_value' if not found.
+        """
+        config = copy.deepcopy(self)
+        if override_configs is not None:
+            config = _recursive_update(config, override_configs)
+        return _get_nested(config, keys, default_value)
+
+    def set_nested(self, keys: Tuple[str, ...], value: Any) -> None:
+        """In-place sets a nested key to value.
+
+        Like get_nested(), if any key is not found, this will not raise an
+        error.
+        """
+        override = {}
+        for i, key in enumerate(reversed(keys)):
+            if i == 0:
+                override = {key: value}
+            else:
+                override = {key: override}
+        _recursive_update(self, override)
+
+    @classmethod
+    def from_dict(cls, config: Optional[Dict[str, Any]]) -> 'Config':
+        if config is None:
+            return cls()
+        return cls(**config)
+
 
 # The loaded config.
-_dict: Optional[Dict[str, Any]] = None
-_loaded_config_path = None
+_dict = Config()
+_loaded_config_path: Optional[str] = None
 
 
 def _get_nested(configs: Optional[Dict[str, Any]], keys: Iterable[str],
@@ -131,17 +179,11 @@ def get_nested(keys: Tuple[str, ...],
     ), (f'Override configs must not be provided when keys {keys} is not within '
         'constants.OVERRIDEABLE_CONFIG_KEYS: '
         f'{constants.OVERRIDEABLE_CONFIG_KEYS}')
-    config: Dict[str, Any] = {}
-    if _dict is not None:
-        config = copy.deepcopy(_dict)
-    if override_configs is None:
-        override_configs = {}
-    config = _recursive_update(config, override_configs)
-    return _get_nested(config, keys, default_value)
+    return _dict.get_nested(keys, default_value, override_configs)
 
 
-def _recursive_update(base_config: Dict[str, Any],
-                      override_config: Dict[str, Any]) -> Dict[str, Any]:
+def _recursive_update(base_config: Config,
+                      override_config: Dict[str, Any]) -> Config:
     """Recursively updates base configuration with override configuration"""
     for key, value in override_config.items():
         if (isinstance(value, dict) and key in base_config and
@@ -157,22 +199,14 @@ def set_nested(keys: Tuple[str, ...], value: Any) -> Dict[str, Any]:
 
     Like get_nested(), if any key is not found, this will not raise an error.
     """
-    _check_loaded_or_die()
-    assert _dict is not None
-    override = {}
-    for i, key in enumerate(reversed(keys)):
-        if i == 0:
-            override = {key: value}
-        else:
-            override = {key: override}
-    return _recursive_update(copy.deepcopy(_dict), override)
+    copied_dict = copy.deepcopy(_dict)
+    copied_dict.set_nested(keys, value)
+    return dict(**copied_dict)
 
 
-def to_dict() -> Dict[str, Any]:
+def to_dict() -> Config:
     """Returns a deep-copied version of the current config."""
-    if _dict is not None:
-        return copy.deepcopy(_dict)
-    return {}
+    return copy.deepcopy(_dict)
 
 
 def _try_load_config() -> None:
@@ -192,13 +226,14 @@ def _try_load_config() -> None:
     config_path = os.path.expanduser(config_path)
     if os.path.exists(config_path):
         logger.debug(f'Using config path: {config_path}')
-        _loaded_config_path = config_path
         try:
-            _dict = common_utils.read_yaml(config_path)
+            config = common_utils.read_yaml(config_path)
+            _dict = Config.from_dict(config)
+            _loaded_config_path = config_path
             logger.debug(f'Config loaded:\n{pprint.pformat(_dict)}')
         except yaml.YAMLError as e:
             logger.error(f'Error in loading config file ({config_path}):', e)
-        if _dict is not None:
+        if _dict:
             common_utils.validate_schema(
                 _dict,
                 schemas.get_config_schema(),
@@ -219,14 +254,6 @@ def loaded_config_path() -> Optional[str]:
 _try_load_config()
 
 
-def _check_loaded_or_die():
-    """Checks loaded() is true; otherwise raises RuntimeError."""
-    if _dict is None:
-        raise RuntimeError(
-            f'No user configs loaded. Check {CONFIG_PATH} exists and '
-            'can be loaded.')
-
-
 def loaded() -> bool:
     """Returns if the user configurations are loaded."""
-    return _dict is not None
+    return bool(_dict)

sky/templates/jobs-controller.yaml.j2

@@ -4,7 +4,9 @@ name: {{dag_name}}
 
 file_mounts:
   {{remote_user_yaml_path}}: {{user_yaml_path}}
-  {{remote_user_config_path}}: skypilot:local_skypilot_config_path
+  {%- if local_user_config_path is not none %}
+  {{remote_user_config_path}}: {{local_user_config_path}}
+  {%- endif %}
   {%- for remote_catalog_path, local_catalog_path in modified_catalogs.items() %}
   {{remote_catalog_path}}: {{local_catalog_path}}
   {%- endfor %}

sky/templates/sky-serve-controller.yaml.j2

@@ -23,7 +23,9 @@ setup: |
 
 file_mounts:
   {{remote_task_yaml_path}}: {{local_task_yaml_path}}
-  {{remote_user_config_path}}: skypilot:local_skypilot_config_path
+  {%- if local_user_config_path is not none %}
+  {{remote_user_config_path}}: {{local_user_config_path}}
+  {%- endif %}
   {%- for remote_catalog_path, local_catalog_path in modified_catalogs.items() %}
   {{remote_catalog_path}}: {{local_catalog_path}}
   {%- endfor %}

sky/utils/admin_policy_utils.py

@@ -0,0 +1,145 @@
+"""Admin policy utils."""
+import copy
+import importlib
+import os
+import tempfile
+from typing import Optional, Tuple, Union
+
+import colorama
+
+from sky import admin_policy
+from sky import dag as dag_lib
+from sky import exceptions
+from sky import sky_logging
+from sky import skypilot_config
+from sky import task as task_lib
+from sky.utils import common_utils
+from sky.utils import ux_utils
+
+logger = sky_logging.init_logger(__name__)
+
+
+def _get_policy_cls(
+        policy: Optional[str]) -> Optional[admin_policy.AdminPolicy]:
+    """Gets admin-defined policy."""
+    if policy is None:
+        return None
+    try:
+        module_path, class_name = policy.rsplit('.', 1)
+        module = importlib.import_module(module_path)
+    except ImportError as e:
+        with ux_utils.print_exception_no_traceback():
+            raise ImportError(
+                f'Failed to import policy module: {policy}. '
+                'Please check if the module is installed in your Python '
+                'environment.') from e
+
+    try:
+        policy_cls = getattr(module, class_name)
+    except AttributeError as e:
+        with ux_utils.print_exception_no_traceback():
+            raise AttributeError(
+                f'Could not find {class_name} class in module {module_path}. '
+                'Please check with your policy admin for details.') from e
+
+    # Check if the module implements the AdminPolicy interface.
+    if not issubclass(policy_cls, admin_policy.AdminPolicy):
+        with ux_utils.print_exception_no_traceback():
+            raise ValueError(
+                f'Policy class {policy!r} does not implement the AdminPolicy '
+                'interface. Please check with your policy admin for details.')
+    return policy_cls
+
+
+def apply(
+    entrypoint: Union['dag_lib.Dag', 'task_lib.Task'],
+    use_mutated_config_in_current_request: bool = True,
+    request_options: Optional[admin_policy.RequestOptions] = None,
+) -> Tuple['dag_lib.Dag', skypilot_config.Config]:
+    """Applies an admin policy (if registered) to a DAG or a task.
+
+    It mutates a Dag by applying any registered admin policy and also
+    potentially updates (controlled by `use_mutated_config_in_current_request`)
+    the global SkyPilot config if there is any changes made by the policy.
+
+    Args:
+        dag: The dag to be mutated by the policy.
+        use_mutated_config_in_current_request: Whether to use the mutated
+            config in the current request.
+        request_options: Additional options user passed for the current request.
+
+    Returns:
+        - The new copy of dag after applying the policy
+        - The new copy of skypilot config after applying the policy.
+    """
+    if isinstance(entrypoint, task_lib.Task):
+        dag = dag_lib.Dag()
+        dag.add(entrypoint)
+    else:
+        dag = entrypoint
+
+    policy = skypilot_config.get_nested(('admin_policy',), None)
+    policy_cls = _get_policy_cls(policy)
+    if policy_cls is None:
+        return dag, skypilot_config.to_dict()
+
+    logger.info(f'Applying policy: {policy}')
+    original_config = skypilot_config.to_dict()
+    config = copy.deepcopy(original_config)
+    mutated_dag = dag_lib.Dag()
+    mutated_dag.name = dag.name
+
+    mutated_config = None
+    for task in dag.tasks:
+        user_request = admin_policy.UserRequest(task, config, request_options)
+        try:
+            mutated_user_request = policy_cls.validate_and_mutate(user_request)
+        except Exception as e:  # pylint: disable=broad-except
+            with ux_utils.print_exception_no_traceback():
+                raise exceptions.UserRequestRejectedByPolicy(
+                    f'{colorama.Fore.RED}User request rejected by policy '
+                    f'{policy!r}{colorama.Fore.RESET}: '
+                    f'{common_utils.format_exception(e, use_bracket=True)}'
+                ) from e
+        if mutated_config is None:
+            mutated_config = mutated_user_request.skypilot_config
+        else:
+            if mutated_config != mutated_user_request.skypilot_config:
+                # In the case of a pipeline of tasks, the mutated config
+                # generated should remain the same for all tasks for now for
+                # simplicity.
+                # TODO(zhwu): We should support per-task mutated config or
+                # allowing overriding required global config in task YAML.
+                with ux_utils.print_exception_no_traceback():
+                    raise exceptions.UserRequestRejectedByPolicy(
+                        'All tasks must have the same SkyPilot config after '
+                        'applying the policy. Please check with your policy '
+                        'admin for details.')
+        mutated_dag.add(mutated_user_request.task)
+    assert mutated_config is not None, dag
+
+    # Update the new_dag's graph with the old dag's graph
+    for u, v in dag.graph.edges:
+        u_idx = dag.tasks.index(u)
+        v_idx = dag.tasks.index(v)
+        mutated_dag.graph.add_edge(mutated_dag.tasks[u_idx],
+                                   mutated_dag.tasks[v_idx])
+
+    if (use_mutated_config_in_current_request and
+            original_config != mutated_config):
+        with tempfile.NamedTemporaryFile(
+                delete=False,
+                mode='w',
+                prefix='policy-mutated-skypilot-config-',
+                suffix='.yaml') as temp_file:
+
+            common_utils.dump_yaml(temp_file.name, dict(**mutated_config))
+            os.environ[skypilot_config.ENV_VAR_SKYPILOT_CONFIG] = temp_file.name
+            logger.debug(f'Updated SkyPilot config: {temp_file.name}')
+            # TODO(zhwu): This is not a clean way to update the SkyPilot config,
+            # because we are resetting the global context for a single DAG,
+            # which is conceptually weird.
+            importlib.reload(skypilot_config)
+
+    logger.debug(f'Mutated user request: {mutated_user_request}')
+    return mutated_dag, mutated_config

sky/utils/common_utils.py

@@ -300,7 +300,7 @@ def user_and_hostname_hash() -> str:
     return f'{getpass.getuser()}-{hostname_hash}'
 
 
-def read_yaml(path) -> Dict[str, Any]:
+def read_yaml(path: str) -> Dict[str, Any]:
     with open(path, 'r', encoding='utf-8') as f:
         config = yaml.safe_load(f)
     return config
@@ -316,12 +316,13 @@ def read_yaml_all(path: str) -> List[Dict[str, Any]]:
         return configs
 
 
-def dump_yaml(path, config) -> None:
+def dump_yaml(path: str, config: Union[List[Dict[str, Any]],
+                                       Dict[str, Any]]) -> None:
     with open(path, 'w', encoding='utf-8') as f:
         f.write(dump_yaml_str(config))
 
 
-def dump_yaml_str(config):
+def dump_yaml_str(config: Union[List[Dict[str, Any]], Dict[str, Any]]) -> str:
     # https://github.com/yaml/pyyaml/issues/127
     class LineBreakDumper(yaml.SafeDumper):
 
@@ -331,9 +332,9 @@ def dump_yaml_str(config):
                 super().write_line_break()
 
     if isinstance(config, list):
-        dump_func = yaml.dump_all
+        dump_func = yaml.dump_all  # type: ignore
     else:
-        dump_func = yaml.dump
+        dump_func = yaml.dump  # type: ignore
     return dump_func(config,
                      Dumper=LineBreakDumper,
                      sort_keys=False,

sky/utils/controller_utils.py

@@ -44,8 +44,12 @@ CONTROLLER_RESOURCES_NOT_VALID_MESSAGE = (
     '{controller_type}.controller.resources is a valid resources spec. '
     'Details:\n  {err}')
 
-# The placeholder for the local skypilot config path in file mounts.
-LOCAL_SKYPILOT_CONFIG_PATH_PLACEHOLDER = 'skypilot:local_skypilot_config_path'
+# The suffix for local skypilot config path for a job/service in file mounts
+# that tells the controller logic to update the config with specific settings,
+# e.g., removing the ssh_proxy_command when a job/service is launched in a same
+# cloud as controller.
+_LOCAL_SKYPILOT_CONFIG_PATH_SUFFIX = (
+    '__skypilot:local_skypilot_config_path.yaml')
 
 
 @dataclasses.dataclass
@@ -350,8 +354,21 @@ def download_and_stream_latest_job_log(
 
 
 def shared_controller_vars_to_fill(
-        controller: Controllers,
-        remote_user_config_path: str) -> Dict[str, str]:
+        controller: Controllers, remote_user_config_path: str,
+        local_user_config: Dict[str, Any]) -> Dict[str, str]:
+    if not local_user_config:
+        local_user_config_path = None
+    else:
+        # Remove admin_policy from local_user_config so that it is not applied
+        # again on the controller. This is required since admin_policy is not
+        # installed on the controller.
+        local_user_config.pop('admin_policy', None)
+        with tempfile.NamedTemporaryFile(
+                delete=False,
+                suffix=_LOCAL_SKYPILOT_CONFIG_PATH_SUFFIX) as temp_file:
+            common_utils.dump_yaml(temp_file.name, dict(**local_user_config))
+        local_user_config_path = temp_file.name
+
     vars_to_fill: Dict[str, Any] = {
         'cloud_dependencies_installation_commands':
             _get_cloud_dependencies_installation_commands(controller),
@@ -360,6 +377,7 @@ def shared_controller_vars_to_fill(
         # accessed.
         'sky_activate_python_env': constants.ACTIVATE_SKY_REMOTE_PYTHON_ENV,
         'sky_python_cmd': constants.SKY_PYTHON_CMD,
+        'local_user_config_path': local_user_config_path,
     }
     env_vars: Dict[str, str] = {
         env.value: '1' for env in env_options.Options if env.get()
@@ -481,7 +499,8 @@ def get_controller_resources(
 
 
 def _setup_proxy_command_on_controller(
-        controller_launched_cloud: 'clouds.Cloud') -> Dict[str, Any]:
+        controller_launched_cloud: 'clouds.Cloud',
+        user_config: Dict[str, Any]) -> skypilot_config.Config:
     """Sets up proxy command on the controller.
 
     This function should be called on the controller (remote cluster), which
@@ -515,21 +534,20 @@ def _setup_proxy_command_on_controller(
     # (or name). It may not be a sufficient check (as it's always
     # possible that peering is not set up), but it may catch some
     # obvious errors.
+    config = skypilot_config.Config.from_dict(user_config)
     proxy_command_key = (str(controller_launched_cloud).lower(),
                          'ssh_proxy_command')
-    ssh_proxy_command = skypilot_config.get_nested(proxy_command_key, None)
-    config_dict = skypilot_config.to_dict()
+    ssh_proxy_command = config.get_nested(proxy_command_key, None)
     if isinstance(ssh_proxy_command, str):
-        config_dict = skypilot_config.set_nested(proxy_command_key, None)
+        config.set_nested(proxy_command_key, None)
     elif isinstance(ssh_proxy_command, dict):
         # Instead of removing the key, we set the value to empty string
         # so that the controller will only try the regions specified by
         # the keys.
         ssh_proxy_command = {k: None for k in ssh_proxy_command}
-        config_dict = skypilot_config.set_nested(proxy_command_key,
-                                                 ssh_proxy_command)
+        config.set_nested(proxy_command_key, ssh_proxy_command)
 
-    return config_dict
+    return config
 
 
 def replace_skypilot_config_path_in_file_mounts(
@@ -543,25 +561,20 @@ def replace_skypilot_config_path_in_file_mounts(
     if file_mounts is None:
         return
     replaced = False
-    to_replace = True
-    with tempfile.NamedTemporaryFile('w', delete=False) as f:
-        if skypilot_config.loaded():
-            new_skypilot_config = _setup_proxy_command_on_controller(cloud)
-            common_utils.dump_yaml(f.name, new_skypilot_config)
-            to_replace = True
-        else:
-            # Empty config. Remove the placeholder below.
-            to_replace = False
-        for remote_path, local_path in list(file_mounts.items()):
-            if local_path == LOCAL_SKYPILOT_CONFIG_PATH_PLACEHOLDER:
-                if to_replace:
-                    file_mounts[remote_path] = f.name
-                    replaced = True
-                else:
-                    del file_mounts[remote_path]
+    for remote_path, local_path in list(file_mounts.items()):
+        if local_path is None:
+            del file_mounts[remote_path]
+            continue
+        if local_path.endswith(_LOCAL_SKYPILOT_CONFIG_PATH_SUFFIX):
+            with tempfile.NamedTemporaryFile('w', delete=False) as f:
+                user_config = common_utils.read_yaml(local_path)
+                config = _setup_proxy_command_on_controller(cloud, user_config)
+                common_utils.dump_yaml(f.name, dict(**config))
+                file_mounts[remote_path] = f.name
+                replaced = True
     if replaced:
-        logger.debug(f'Replaced {LOCAL_SKYPILOT_CONFIG_PATH_PLACEHOLDER} with '
-                     f'the real path in file mounts: {file_mounts}')
+        logger.debug(f'Replaced {_LOCAL_SKYPILOT_CONFIG_PATH_SUFFIX} '
+                     f'with the real path in file mounts: {file_mounts}')
 
 
 def maybe_translate_local_file_mounts_and_sync_up(task: 'task_lib.Task',

sky/utils/dag_utils.py

@@ -36,30 +36,33 @@ The command can then be run as:
 
 
 def convert_entrypoint_to_dag(entrypoint: Any) -> 'dag_lib.Dag':
-    """Convert the entrypoint to a sky.Dag.
+    """Converts the entrypoint to a sky.Dag and applies the policy.
 
     Raises TypeError if 'entrypoint' is not a 'sky.Task' or 'sky.Dag'.
     """
     # Not suppressing stacktrace: when calling this via API user may want to
     # see their own program in the stacktrace. Our CLI impl would not trigger
     # these errors.
+    converted_dag: 'dag_lib.Dag'
     if isinstance(entrypoint, str):
         with ux_utils.print_exception_no_traceback():
             raise TypeError(_ENTRYPOINT_STRING_AS_DAG_MESSAGE)
     elif isinstance(entrypoint, dag_lib.Dag):
-        return copy.deepcopy(entrypoint)
+        converted_dag = copy.deepcopy(entrypoint)
     elif isinstance(entrypoint, task_lib.Task):
         entrypoint = copy.deepcopy(entrypoint)
         with dag_lib.Dag() as dag:
             dag.add(entrypoint)
             dag.name = entrypoint.name
-        return dag
+        converted_dag = dag
     else:
         with ux_utils.print_exception_no_traceback():
             raise TypeError(
                 'Expected a sky.Task or sky.Dag but received argument of type: '
                 f'{type(entrypoint)}')
 
+    return converted_dag
+
 
 def load_chain_dag_from_yaml(
     path: str,

sky/utils/schemas.py

@@ -848,6 +848,13 @@ def get_config_schema():
         },
     }
 
+    admin_policy_schema = {
+        'type': 'string',
+        # Check regex to be a valid python module path
+        'pattern': (r'^[a-zA-Z_][a-zA-Z0-9_]*'
+                    r'(\.[a-zA-Z_][a-zA-Z0-9_]*)+$'),
+    }
+
     allowed_clouds = {
         # A list of cloud names that are allowed to be used
         'type': 'array',
@@ -905,6 +912,7 @@ def get_config_schema():
             'spot': controller_resources_schema,
             'serve': controller_resources_schema,
             'allowed_clouds': allowed_clouds,
+            'admin_policy': admin_policy_schema,
             'docker': docker_configs,
             'nvidia_gpus': gpu_configs,
             **cloud_configs,

{skypilot_nightly-1.0.0.dev20240923.dist-info → skypilot_nightly-1.0.0.dev20240925.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: skypilot-nightly
-Version: 1.0.0.dev20240923
+Version: 1.0.0.dev20240925
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0

{skypilot_nightly-1.0.0.dev20240923.dist-info → skypilot_nightly-1.0.0.dev20240925.dist-info}/RECORD

@@ -1,17 +1,18 @@
-sky/__init__.py,sha256=ABpdCy_YwMSkcSplyajbIKQGxWNoWGWXAZtJmSGIevs,5588
+sky/__init__.py,sha256=n-Q-Qhc50FJ91_dAC8eBKZ84oIShcdcPKn3u9eWTR7M,5854
+sky/admin_policy.py,sha256=hPo02f_A32gCqhUueF0QYy1fMSSKqRwYEg_9FxScN_s,3248
 sky/authentication.py,sha256=yvpdkXS9htf-X83DPCiSG3mQ41y0zV1BQ0YgOMgTYBU,20612
 sky/check.py,sha256=jLMIIJrseaZj1_o5WkbaD9XdyXIlCaT6pyAaIFdhdmA,9079
 sky/cli.py,sha256=2cOw3lXzRA-uLlEH-eK7N_1VmUpf1LR4Ztu-ZaKu3Is,201673
 sky/cloud_stores.py,sha256=RjFgmRhUh1Kk__f6g3KxzLp9s7dA0pFK4W1AukEuUaw,21153
 sky/core.py,sha256=YF_6kwj8Ja171Oycb8L25SZ7V_ylZYovFS_jpnjwGo0,34408
-sky/dag.py,sha256=d3gF030wYmit01jxszEygT4EvKJUky3DBlG3PfWPp_w,2529
-sky/exceptions.py,sha256=gfrmh8djfZ0oGn1XtYWn8ca7wEyghkbzg4Hkq6tdjj8,8594
-sky/execution.py,sha256=nyLjzYOY1e-NHJeufrx24Oz2_tyBrKVoAu_tQtrhL0I,25235
+sky/dag.py,sha256=WLFWr5hfrwjd31uYlNvI-zWUk7tLaT_gzJn4LzbVtkE,2780
+sky/exceptions.py,sha256=s7j0iCa1Ec0rU1ABb9EAhqn2qFm22bmKQV_ckgRlMGk,8720
+sky/execution.py,sha256=97yhNh5BKBh2ZJW8GefGID_4KCYU-arnvemSJB9rf6U,25552
 sky/global_user_state.py,sha256=PywEmUutF97XBgRMClR6IS5_KM8JJC0oA1LsPUZebp0,28681
 sky/optimizer.py,sha256=YGBhJPlcvylYON7MLrYEMtBOqJLt4LdlguQclVvvl4E,58677
 sky/resources.py,sha256=_959wcQnoiAYesslN9BPXWABFaQfc_TFXPO_o7SPlxI,67325
 sky/sky_logging.py,sha256=I59__M9taBjDim15ie0m25Vtn6itLtR9Ao8W9FS36Xs,4253
-sky/skypilot_config.py,sha256=eyXxUljnLoQRLPO5sqm5MjuydhQIhu6RGPNn5Op0LpI,8063
+sky/skypilot_config.py,sha256=E3g65cX3P3dT9b5N0GgFBG6yB0FXwIGpisKoozmJmWU,9094
 sky/status_lib.py,sha256=J7Jb4_Dz0v2T64ttOdyUgpokvl4S0sBJrMfH7Fvo51A,1457
 sky/task.py,sha256=KDsTBIxYpkCOPHv3_ei5H3LDMiGHvDeS9_2HeL6yyLA,49766
 sky/adaptors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -30,7 +31,7 @@ sky/adaptors/vsphere.py,sha256=zJP9SeObEoLrpgHW2VHvZE48EhgVf8GfAEIwBeaDMfM,2129
 sky/backends/__init__.py,sha256=UDjwbUgpTRApbPJnNfR786GadUuwgRk3vsWoVu5RB_c,536
 sky/backends/backend.py,sha256=xtxR6boDv1o-uSCjbJhOMkKMnZvBZh3gExx4khFWPTI,5932
 sky/backends/backend_utils.py,sha256=W11gOb3v6Z2PLFu5YbFHDCckjgfwiSieBiDtmMeJXpE,126590
-sky/backends/cloud_vm_ray_backend.py,sha256=do6bEyVKxojkF8zIC4AI3lSrjREbxteOkxqeXdplSi4,232350
+sky/backends/cloud_vm_ray_backend.py,sha256=3bPAln6pAP5r_Xm1_d2whInS6wO_fh1i7G0heGxvjY8,232893
 sky/backends/docker_utils.py,sha256=Hyw1YY20EyghhEbYx6O2FIMDcGkNzBzV9TM7LFynei8,8358
 sky/backends/local_docker_backend.py,sha256=H4GBo0KFUC_EEf-ziv1OUbfAkOI5BrwkYs9fYOxSoNw,16741
 sky/backends/wheel_utils.py,sha256=3QS4T_Ydvo4DbYhogtyADyNBEf04I6jUCL71M285shQ,7963
@@ -94,8 +95,8 @@ sky/data/storage.py,sha256=JYeEWQtCXg9h6CjCaAsINV1MXcvFswuUgf3c0GOZQSs,162523
 sky/data/storage_utils.py,sha256=-s0iQhV8JVx1J2gWtoBFrN04MGv2oVYxo_Hw43R2BSY,6867
 sky/jobs/__init__.py,sha256=9cqFutVlfjQb7t8hzG-ZlQmMlbmfMirn0KNBxIFnJYQ,1398
 sky/jobs/constants.py,sha256=YLgcCg_RHSYr_rfsI_4UIdXk78KKKOK29Oem88t5j8I,1350
-sky/jobs/controller.py,sha256=VbEUusR85Mady-lJhf7jwliydiBewvlLlPbFcjOEiMU,26607
-sky/jobs/core.py,sha256=Be_RglG5vzJCf1tj9vLcOpkCjzCGfyY7it0EcjMX8lg,13426
+sky/jobs/controller.py,sha256=k28bbicxtML6p1YxSetk-1nhBHPCubpvLWJsh7TtU9c,26701
+sky/jobs/core.py,sha256=Q5ExRWnF7yAYWJxwnB9NfAGBVDNqKYBCrWsypiMLCpY,13637
 sky/jobs/recovery_strategy.py,sha256=G3iFicEajB-l9FefvcqjqPIazb1X8BJ_AgVmD5bDV2w,25556
 sky/jobs/state.py,sha256=nf6hduj0dSekmP7scW5mKHiYQs6WkdigJpJJbLEBdEw,23041
 sky/jobs/utils.py,sha256=ZB2dJxtJ4hbCRdxHmy8wrmtXIvvGGE80kk5BQTOQWkQ,35653
@@ -171,7 +172,7 @@ sky/serve/__init__.py,sha256=Qg_XPOtQsUxiN-Q3njHZRfzoMcQ_KKU1QthkiTbESDw,1661
 sky/serve/autoscalers.py,sha256=CAdp0vFN_eBx8K3svJToj1hTEc29M5gB2V1BhV9a_lI,30136
 sky/serve/constants.py,sha256=OansIC7a0Pwat-Y5SF43T9phad_EvyjKO3peZgKFEHk,4367
 sky/serve/controller.py,sha256=spXd-QXOy0xpyN5VtZ6E3BoHIvE3fOZTysKqS-X880U,7651
-sky/serve/core.py,sha256=NDs2fRRRHtsI-ydokSj3VJuj9nwKTrVeCqhMpAQ0-j0,28710
+sky/serve/core.py,sha256=cW2SNMPMbGtOcqASHnL__B12BCIErUilGtFw3olQbjk,28947
 sky/serve/load_balancer.py,sha256=90HWkhhmfrW9kaGVSMxnqzYB_o5PwAOKX0mTAVfQVFM,11548
 sky/serve/load_balancing_policies.py,sha256=ExdwH_pxPYpJ6CkoTQCOPSa4lzwbq1LFFMKzmIu8ryk,2331
 sky/serve/replica_managers.py,sha256=c-S8m1Akgy13MjjX3BrqRlUiwbpjbicXEyWASjosPN0,57308
@@ -222,7 +223,7 @@ sky/templates/cudo-ray.yml.j2,sha256=SEHVY57iBauCOE2HYJtYVFEKlriAkdwQu_p86a1n_bA
 sky/templates/fluidstack-ray.yml.j2,sha256=t8TCULgiErCZdtFmBZVsA8ZdcqR7ccwsmQhuDFTBEAU,3541
 sky/templates/gcp-ray.yml.j2,sha256=q2xSWxxYI8MVAq_mA__8FF6PwEqXCAW1SOEOGTt0qPw,9591
 sky/templates/ibm-ray.yml.j2,sha256=RMBUqPId8i4CnVwcyfK3DbRapF1jFMuGQlY0E0PFbMU,6669
-sky/templates/jobs-controller.yaml.j2,sha256=QjlIcFBEay48xKT50UWqzgREzili-H7AuUrnGHWOpI8,1554
+sky/templates/jobs-controller.yaml.j2,sha256=Gu3ogFxFYr09VEXP-6zEbrCUOFo1aYxWEjAq7whCrxo,1607
 sky/templates/kubernetes-ingress.yml.j2,sha256=73iDklVDWBMbItg0IexCa6_ClXPJOxw7PWz3leku4nE,1340
 sky/templates/kubernetes-loadbalancer.yml.j2,sha256=IxrNYM366N01bbkJEbZ_UPYxUP8wyVEbRNFHRsBuLsw,626
 sky/templates/kubernetes-port-forward-proxy-command.sh,sha256=HlG7CPBBedCVBlL9qv0erW_eKm6Irj0LFyaAWuJW_lc,3148
@@ -234,26 +235,27 @@ sky/templates/oci-ray.yml.j2,sha256=5XfIobW9XuspIpEhI4vFIEcJEFCdtFJqEGfX03zL6DE,
 sky/templates/paperspace-ray.yml.j2,sha256=HQjZNamrB_a4fOMCxQXSVdV5JIHtbGtAE0JzEO8uuVQ,4021
 sky/templates/runpod-ray.yml.j2,sha256=p3BtYBHzROtNJqnjEo1xCmGSJQfCZYdarWszhDYyl0Q,3697
 sky/templates/scp-ray.yml.j2,sha256=I9u8Ax-lit-d6UrCC9BVU8avst8w1cwK6TrzZBcz_JM,5608
-sky/templates/sky-serve-controller.yaml.j2,sha256=IWNdhBSoCLLE4GG87ztk_P0EmZ6LS_CHHgWFVzOph5c,1575
+sky/templates/sky-serve-controller.yaml.j2,sha256=V1IiYhArv_D_7JzC3sVN4nKlSCCCL1AYtIdxFSa-f4c,1628
 sky/templates/vsphere-ray.yml.j2,sha256=cOQ-qdpxGA2FHajMMhTJI-SmlYzdPterX4Gsiq-nkb0,3587
 sky/usage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/usage/constants.py,sha256=8xpg9vhDU9A3eObtpkNFjwa42oCazqGEv4yw_vJSO7U,590
 sky/usage/usage_lib.py,sha256=uqclBc87_9D_QVWigCjOIfWFoVB6re68C7RnwjzRYvg,17870
 sky/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/utils/accelerator_registry.py,sha256=BO4iYH5bV80Xyp4EPfO0n1D3LL0FvESCy7xm59Je3_o,3798
+sky/utils/admin_policy_utils.py,sha256=zFCu1OFIrZRfQNY0JFRO1502WFfdqZhwAU_QgM4fO9U,5943
 sky/utils/cluster_yaml_utils.py,sha256=1wRRYqI1kI-eFs1pMW4r_FFjHJ0zamq6v2RRI-Gtx5E,849
 sky/utils/command_runner.py,sha256=SKA90-i4FK7fn9TpkztXY6E3fCBGUjGj-jcrgvb8DAQ,33643
 sky/utils/command_runner.pyi,sha256=1khh14BhdOpMxvk9Ydnd3OFdas5Nha6dSOzy5xLBUU4,7710
-sky/utils/common_utils.py,sha256=HMCYoxOxLsgdKyRJW9PZHjE3fZ-CmYMSk-jpiINSduY,23850
-sky/utils/controller_utils.py,sha256=z5eclkS67dNz54KyHtftzu-HV7O1XPnURgelRY9N63s,36783
-sky/utils/dag_utils.py,sha256=ddSmnOjE9yMpaSC5jlC-x_tq1hPsVYJzBTyDch_cpr4,5487
+sky/utils/common_utils.py,sha256=O6PlZTCNhbuXOzjuV2DKw43niWE_qPfYZNGhnMtZzQg,24028
+sky/utils/controller_utils.py,sha256=VtdjKH9u1kWwUOMzPUxuLpT-XXQ2gCLCLOldB-vdh_8,37483
+sky/utils/dag_utils.py,sha256=gjGZiJj4_GYsraXX67e6ElvbmOByJcyjSfvVgYZiXvs,5588
 sky/utils/db_utils.py,sha256=AOvMmBEN9cF4I7CoXihPCtus4mU2VDGjBQSVMMgzKlA,2786
 sky/utils/env_options.py,sha256=1VXyd3bhiUgGfCpmmTqM9PagRo1ILBH4-pzIxmIeE6E,861
 sky/utils/kubernetes_enums.py,sha256=imGqHSa8O07zD_6xH1SDMM7dBU5lF5fzFFlQuQy00QM,1384
 sky/utils/log_utils.py,sha256=W7FYK7xzvbq4V-8R-ihLtz939ryvtABug6O-4DFrjho,8139
 sky/utils/resources_utils.py,sha256=snByBxgx3Hnjfch2uysdAA3D-OAwrnuzTDHug36s5H4,6515
 sky/utils/rich_utils.py,sha256=5ZVhzlFx-nhqMXwv00eO9xC4rz7ibDlfD2lmGhZrJEY,1581
-sky/utils/schemas.py,sha256=dGjxAXpaVewRZ7vh4vLIAnOiflOPActnc1LVJJ5RMoU,28331
+sky/utils/schemas.py,sha256=bJan9aOi2gK6jnTzxNkX1kivIUsQ5Mc5SzKiBj3A97g,28597
 sky/utils/subprocess_utils.py,sha256=zK0L3mvAkvKX1nFFI4IFEmRWX9ytpguhhxOTYUDKoDs,6507
 sky/utils/timeline.py,sha256=ao_nm0y52ZQILfL7Y92c3pSEFRyPm_ElORC3DrI5BwQ,3936
 sky/utils/ux_utils.py,sha256=318TRunQCyJpJXonfiJ1SVotNA-6K4F2XgMEYjvWvsk,3264
@@ -270,9 +272,9 @@ sky/utils/kubernetes/k8s_gpu_labeler_job.yaml,sha256=KPqp23B-zQ2SZK03jdHeF9fLTog
 sky/utils/kubernetes/k8s_gpu_labeler_setup.yaml,sha256=VLKT2KKimZu1GDg_4AIlIt488oMQvhRZWwsj9vBbPUg,3812
 sky/utils/kubernetes/rsync_helper.sh,sha256=Ma-N9a271fTfdgP5-8XIQL7KPf8IPUo-uY004PCdUFo,747
 sky/utils/kubernetes/ssh_jump_lifecycle_manager.py,sha256=RFLJ3k7MR5UN4SKHykQ0lV9SgXumoULpKYIAt1vh-HU,6560
-skypilot_nightly-1.0.0.dev20240923.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
-skypilot_nightly-1.0.0.dev20240923.dist-info/METADATA,sha256=sZYi5_sZTkL-XXdjdk8XEsAFCy3zMgzhTHlIFI_TS7s,19011
-skypilot_nightly-1.0.0.dev20240923.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-skypilot_nightly-1.0.0.dev20240923.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
-skypilot_nightly-1.0.0.dev20240923.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
-skypilot_nightly-1.0.0.dev20240923.dist-info/RECORD,,
+skypilot_nightly-1.0.0.dev20240925.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
+skypilot_nightly-1.0.0.dev20240925.dist-info/METADATA,sha256=ed5Jd2XESpKbZee7szKC7QnUHl8s73cNB0iEgdJrfpk,19011
+skypilot_nightly-1.0.0.dev20240925.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+skypilot_nightly-1.0.0.dev20240925.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
+skypilot_nightly-1.0.0.dev20240925.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
+skypilot_nightly-1.0.0.dev20240925.dist-info/RECORD,,