skypilot-nightly 1.0.0.dev20250303__py3-none-any.whl → 1.0.0.dev20250304__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '50bbf9162d3516d5eb4e59e071822a4f3c38622c'
+_SKYPILOT_COMMIT_SHA = '5c126400a3a3791b34beebd5e7b5a3717efd9505'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250303'
+__version__ = '1.0.0.dev20250304'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

sky/backends/cloud_vm_ray_backend.py

@@ -3823,6 +3823,10 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
             follow: Whether to follow the logs.
             tail: The number of lines to display from the end of the
                 log file. If 0, print all lines.
+
+        Returns:
+            The exit code of the tail command. Returns code 100 if the job has
+            failed. See exceptions.JobExitCode for possible return codes.
         """
         code = job_lib.JobLibCodeGen.tail_logs(job_id,
                                                managed_job_id=managed_job_id,
@@ -3856,7 +3860,7 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
                               job_id: Optional[int] = None,
                               job_name: Optional[str] = None,
                               controller: bool = False,
-                              follow: bool = True) -> None:
+                              follow: bool = True) -> int:
         # if job_name is not None, job_id should be None
         assert job_name is None or job_id is None, (job_name, job_id)
         code = managed_jobs.ManagedJobCodeGen.stream_logs(
@@ -3869,13 +3873,17 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
             signal.signal(signal.SIGTSTP, backend_utils.stop_handler)
 
         # Refer to the notes in tail_logs.
-        self.run_on_head(
-            handle,
-            code,
-            stream_logs=True,
-            process_stream=False,
-            ssh_mode=command_runner.SshMode.INTERACTIVE,
-        )
+        try:
+            returncode = self.run_on_head(
+                handle,
+                code,
+                stream_logs=True,
+                process_stream=False,
+                ssh_mode=command_runner.SshMode.INTERACTIVE,
+            )
+        except SystemExit as e:
+            returncode = e.code
+        return returncode
 
     def sync_down_managed_job_logs(
             self,

sky/cli.py

@@ -1227,11 +1227,15 @@ def launch(
             clusters=[handle.get_cluster_name()])
         # job_id will be None if no job was submitted (e.g. no entrypoint
         # provided)
+        returncode = 0
         if not detach_run and job_id is not None:
-            sdk.tail_logs(handle.get_cluster_name(), job_id, follow=True)
+            returncode = sdk.tail_logs(handle.get_cluster_name(),
+                                       job_id,
+                                       follow=True)
         click.secho(
             ux_utils.command_hint_messages(ux_utils.CommandHintType.CLUSTER_JOB,
                                            job_id, handle.get_cluster_name()))
+        sys.exit(returncode)
 
 
 @cli.command(cls=_DocumentedCodeCommand)
@@ -1377,7 +1381,8 @@ def exec(cluster: Optional[str], cluster_option: Optional[str],
     job_id_handle = _async_call_or_wait(request_id, async_call, 'sky.exec')
     if not async_call and not detach_run:
         job_id, _ = job_id_handle
-        sdk.tail_logs(cluster, job_id, follow=True)
+        returncode = sdk.tail_logs(cluster, job_id, follow=True)
+        sys.exit(returncode)
 
 
 def _handle_jobs_queue_request(
@@ -2121,12 +2126,20 @@ def logs(
     one job_id can be provided.
 
     2. If ``--status`` is specified, print the status of the job and exit with
-    returncode 0 if the job succeeded, or 1 otherwise. At most one job_id can
-    be specified.
+    returncode 0 if the job succeeded. At most one job_id can
+    be specified. Other possible return codes:
+
+    - 100: job failed.
+    - 101: job not finished.
+    - 102: job not found.
+    - 103: job was cancelled by the user.
 
     3. If ``--sync-down`` is specified, the logs of the job will be downloaded
     from the cluster and saved to the local machine under
-    ``~/sky_logs``. Mulitple job_ids can be specified.
+    ``~/sky_logs``. Multiple job_ids can be specified.
+
+    4. If the job fails or fetching the logs fails, the command will exit with
+    a non-zero return code.
     """
     if sync_down and status:
         raise click.UsageError(
@@ -2174,17 +2187,18 @@ def logs(
         # it will return {None: None}.
         if job_id is None:
             click.secho(f'No job found on cluster {cluster!r}.', fg='red')
-            sys.exit(1)
+            sys.exit(exceptions.JobExitCode.NOT_FOUND)
         job_status = list(job_statuses.values())[0]
         job_status_str = job_status.value if job_status is not None else 'None'
         click.echo(f'Job {job_id}: {job_status_str}')
         if job_status == job_lib.JobStatus.SUCCEEDED:
             return
         else:
+            returncode = exceptions.JobExitCode.from_job_status(job_status)
             if job_status is None:
                 id_str = '' if job_id is None else f'{job_id} '
                 click.secho(f'Job {id_str}not found', fg='red')
-            sys.exit(1)
+            sys.exit(returncode)
 
     job_str = f'job {job_id}'
     if job_id is None:
@@ -2194,7 +2208,8 @@ def logs(
                 f'{colorama.Style.RESET_ALL}')
 
     # Stream logs from the server.
-    sdk.tail_logs(cluster, job_id, follow, tail=tail)
+    returncode = sdk.tail_logs(cluster, job_id, follow, tail=tail)
+    sys.exit(returncode)
 
 
 @cli.command()
@@ -3729,6 +3744,7 @@ def storage_delete(names: List[str], all: bool, yes: bool, async_call: bool):  #
         if not storages:
             click.echo('No storage(s) to delete.')
             return
+        names = [storage['name'] for storage in storages]
     else:
         names = _get_glob_storages(names)
     if names:
@@ -3893,10 +3909,11 @@ def jobs_launch(
                                         'sky.jobs.launch')
     if not async_call and not detach_run:
         job_id = job_id_handle[0]
-        managed_jobs.tail_logs(name=None,
-                               job_id=job_id,
-                               follow=True,
-                               controller=False)
+        returncode = managed_jobs.tail_logs(name=None,
+                                            job_id=job_id,
+                                            follow=True,
+                                            controller=False)
+        sys.exit(returncode)
 
 
 @jobs.command('queue', cls=_DocumentedCodeCommand)
@@ -4127,11 +4144,12 @@ def jobs_logs(name: Optional[str], job_id: Optional[int], follow: bool,
                 logger.info(f'{fore.CYAN}Job {job} logs{controller_str}: '
                             f'{log_local_path}{style.RESET_ALL}')
         else:
-            managed_jobs.tail_logs(name=name,
-                                   job_id=job_id,
-                                   follow=follow,
-                                   controller=controller,
-                                   refresh=refresh)
+            returncode = managed_jobs.tail_logs(name=name,
+                                                job_id=job_id,
+                                                follow=follow,
+                                                controller=controller,
+                                                refresh=refresh)
+            sys.exit(returncode)
     except exceptions.ClusterNotUpError:
         with ux_utils.print_exception_no_traceback():
             raise

sky/client/cli.py

@@ -1227,11 +1227,15 @@ def launch(
             clusters=[handle.get_cluster_name()])
         # job_id will be None if no job was submitted (e.g. no entrypoint
         # provided)
+        returncode = 0
         if not detach_run and job_id is not None:
-            sdk.tail_logs(handle.get_cluster_name(), job_id, follow=True)
+            returncode = sdk.tail_logs(handle.get_cluster_name(),
+                                       job_id,
+                                       follow=True)
         click.secho(
             ux_utils.command_hint_messages(ux_utils.CommandHintType.CLUSTER_JOB,
                                            job_id, handle.get_cluster_name()))
+        sys.exit(returncode)
 
 
 @cli.command(cls=_DocumentedCodeCommand)
@@ -1377,7 +1381,8 @@ def exec(cluster: Optional[str], cluster_option: Optional[str],
     job_id_handle = _async_call_or_wait(request_id, async_call, 'sky.exec')
     if not async_call and not detach_run:
         job_id, _ = job_id_handle
-        sdk.tail_logs(cluster, job_id, follow=True)
+        returncode = sdk.tail_logs(cluster, job_id, follow=True)
+        sys.exit(returncode)
 
 
 def _handle_jobs_queue_request(
@@ -2121,12 +2126,20 @@ def logs(
     one job_id can be provided.
 
     2. If ``--status`` is specified, print the status of the job and exit with
-    returncode 0 if the job succeeded, or 1 otherwise. At most one job_id can
-    be specified.
+    returncode 0 if the job succeeded. At most one job_id can
+    be specified. Other possible return codes:
+
+    - 100: job failed.
+    - 101: job not finished.
+    - 102: job not found.
+    - 103: job was cancelled by the user.
 
     3. If ``--sync-down`` is specified, the logs of the job will be downloaded
     from the cluster and saved to the local machine under
-    ``~/sky_logs``. Mulitple job_ids can be specified.
+    ``~/sky_logs``. Multiple job_ids can be specified.
+
+    4. If the job fails or fetching the logs fails, the command will exit with
+    a non-zero return code.
     """
     if sync_down and status:
         raise click.UsageError(
@@ -2174,17 +2187,18 @@ def logs(
         # it will return {None: None}.
         if job_id is None:
             click.secho(f'No job found on cluster {cluster!r}.', fg='red')
-            sys.exit(1)
+            sys.exit(exceptions.JobExitCode.NOT_FOUND)
         job_status = list(job_statuses.values())[0]
         job_status_str = job_status.value if job_status is not None else 'None'
         click.echo(f'Job {job_id}: {job_status_str}')
         if job_status == job_lib.JobStatus.SUCCEEDED:
             return
         else:
+            returncode = exceptions.JobExitCode.from_job_status(job_status)
             if job_status is None:
                 id_str = '' if job_id is None else f'{job_id} '
                 click.secho(f'Job {id_str}not found', fg='red')
-            sys.exit(1)
+            sys.exit(returncode)
 
     job_str = f'job {job_id}'
     if job_id is None:
@@ -2194,7 +2208,8 @@ def logs(
                 f'{colorama.Style.RESET_ALL}')
 
     # Stream logs from the server.
-    sdk.tail_logs(cluster, job_id, follow, tail=tail)
+    returncode = sdk.tail_logs(cluster, job_id, follow, tail=tail)
+    sys.exit(returncode)
 
 
 @cli.command()
@@ -3729,6 +3744,7 @@ def storage_delete(names: List[str], all: bool, yes: bool, async_call: bool):  #
         if not storages:
             click.echo('No storage(s) to delete.')
             return
+        names = [storage['name'] for storage in storages]
     else:
         names = _get_glob_storages(names)
     if names:
@@ -3893,10 +3909,11 @@ def jobs_launch(
                                         'sky.jobs.launch')
     if not async_call and not detach_run:
         job_id = job_id_handle[0]
-        managed_jobs.tail_logs(name=None,
-                               job_id=job_id,
-                               follow=True,
-                               controller=False)
+        returncode = managed_jobs.tail_logs(name=None,
+                                            job_id=job_id,
+                                            follow=True,
+                                            controller=False)
+        sys.exit(returncode)
 
 
 @jobs.command('queue', cls=_DocumentedCodeCommand)
@@ -4127,11 +4144,12 @@ def jobs_logs(name: Optional[str], job_id: Optional[int], follow: bool,
                 logger.info(f'{fore.CYAN}Job {job} logs{controller_str}: '
                             f'{log_local_path}{style.RESET_ALL}')
         else:
-            managed_jobs.tail_logs(name=name,
-                                   job_id=job_id,
-                                   follow=follow,
-                                   controller=controller,
-                                   refresh=refresh)
+            returncode = managed_jobs.tail_logs(name=name,
+                                                job_id=job_id,
+                                                follow=follow,
+                                                controller=controller,
+                                                refresh=refresh)
+            sys.exit(returncode)
     except exceptions.ClusterNotUpError:
         with ux_utils.print_exception_no_traceback():
             raise

sky/client/sdk.py

@@ -25,6 +25,7 @@ import filelock
 import psutil
 import requests
 
+from sky import admin_policy
 from sky import backends
 from sky import exceptions
 from sky import sky_logging
@@ -212,13 +213,17 @@ def list_accelerator_counts(
 @annotations.client_api
 def optimize(
     dag: 'sky.Dag',
-    minimize: common.OptimizeTarget = common.OptimizeTarget.COST
+    minimize: common.OptimizeTarget = common.OptimizeTarget.COST,
+    admin_policy_request_options: Optional[admin_policy.RequestOptions] = None
 ) -> server_common.RequestId:
     """Finds the best execution plan for the given DAG.
 
     Args:
         dag: the DAG to optimize.
         minimize: whether to minimize cost or time.
+        admin_policy_request_options: Request options used for admin policy
+            validation. This is only required when a admin policy is in use,
+            see: https://docs.skypilot.co/en/latest/cloud-setup/policy.html
 
     Returns:
         The request ID of the optimize request.
@@ -233,7 +238,9 @@ def optimize(
     """
     dag_str = dag_utils.dump_chain_dag_to_yaml_str(dag)
 
-    body = payloads.OptimizeBody(dag=dag_str, minimize=minimize)
+    body = payloads.OptimizeBody(dag=dag_str,
+                                 minimize=minimize,
+                                 request_options=admin_policy_request_options)
     response = requests.post(f'{server_common.get_server_url()}/optimize',
                              json=json.loads(body.model_dump_json()))
     return server_common.get_request_id(response)
@@ -242,7 +249,11 @@ def optimize(
 @usage_lib.entrypoint
 @server_common.check_server_healthy_or_start
 @annotations.client_api
-def validate(dag: 'sky.Dag', workdir_only: bool = False) -> None:
+def validate(
+    dag: 'sky.Dag',
+    workdir_only: bool = False,
+    admin_policy_request_options: Optional[admin_policy.RequestOptions] = None
+) -> None:
     """Validates the tasks.
 
     The file paths (workdir and file_mounts) are validated on the client side
@@ -254,13 +265,17 @@ def validate(dag: 'sky.Dag', workdir_only: bool = False) -> None:
         dag: the DAG to validate.
         workdir_only: whether to only validate the workdir. This is used for
             `exec` as it does not need other files/folders in file_mounts.
+        admin_policy_request_options: Request options used for admin policy
+            validation. This is only required when a admin policy is in use,
+            see: https://docs.skypilot.co/en/latest/cloud-setup/policy.html
     """
     for task in dag.tasks:
         task.expand_and_validate_workdir()
         if not workdir_only:
             task.expand_and_validate_file_mounts()
     dag_str = dag_utils.dump_chain_dag_to_yaml_str(dag)
-    body = payloads.ValidateBody(dag=dag_str)
+    body = payloads.ValidateBody(dag=dag_str,
+                                 request_options=admin_policy_request_options)
     response = requests.post(f'{server_common.get_server_url()}/validate',
                              json=json.loads(body.model_dump_json()))
     if response.status_code == 400:
@@ -386,7 +401,12 @@ def launch(
                                       'Please contact the SkyPilot team if you '
                                       'need this feature at slack.skypilot.co.')
     dag = dag_utils.convert_entrypoint_to_dag(task)
-    validate(dag)
+    request_options = admin_policy.RequestOptions(
+        cluster_name=cluster_name,
+        idle_minutes_to_autostop=idle_minutes_to_autostop,
+        down=down,
+        dryrun=dryrun)
+    validate(dag, admin_policy_request_options=request_options)
 
     confirm_shown = False
     if _need_confirmation:
@@ -400,7 +420,8 @@ def launch(
         if not clusters:
             # Show the optimize log before the prompt if the cluster does not
             # exist.
-            request_id = optimize(dag)
+            request_id = optimize(dag,
+                                  admin_policy_request_options=request_options)
             stream_and_get(request_id)
         else:
             cluster_record = clusters[0]
@@ -562,7 +583,7 @@ def tail_logs(cluster_name: str,
               job_id: Optional[int],
               follow: bool,
               tail: int = 0,
-              output_stream: Optional['io.TextIOBase'] = None) -> None:
+              output_stream: Optional['io.TextIOBase'] = None) -> int:
     """Tails the logs of a job.
 
     Args:
@@ -575,7 +596,9 @@ def tail_logs(cluster_name: str,
             console.
 
     Returns:
-        None
+        Exit code based on success or failure of the job. 0 if success,
+        100 if the job failed. See exceptions.JobExitCode for possible exit
+        codes.
 
     Request Raises:
         ValueError: if arguments are invalid or the cluster is not supported.
@@ -601,7 +624,7 @@ def tail_logs(cluster_name: str,
         timeout=(client_common.API_SERVER_REQUEST_CONNECTION_TIMEOUT_SECONDS,
                  None))
     request_id = server_common.get_request_id(response)
-    stream_response(request_id, response, output_stream)
+    return stream_response(request_id, response, output_stream)
 
 
 @usage_lib.entrypoint

sky/core.py

@@ -6,16 +6,18 @@ from typing import Any, Dict, List, Optional, Tuple, Union
 
 import colorama
 
+from sky import admin_policy
 from sky import backends
 from sky import check as sky_check
 from sky import clouds
-from sky import dag
+from sky import dag as dag_lib
 from sky import data
 from sky import exceptions
 from sky import global_user_state
 from sky import models
+from sky import optimizer
 from sky import sky_logging
-from sky import task
+from sky import task as task_lib
 from sky.backends import backend_utils
 from sky.clouds import service_catalog
 from sky.jobs.server import core as managed_jobs_core
@@ -25,6 +27,7 @@ from sky.skylet import constants
 from sky.skylet import job_lib
 from sky.skylet import log_lib
 from sky.usage import usage_lib
+from sky.utils import admin_policy_utils
 from sky.utils import common
 from sky.utils import common_utils
 from sky.utils import controller_utils
@@ -44,6 +47,46 @@ logger = sky_logging.init_logger(__name__)
 # ======================
 
 
+@usage_lib.entrypoint
+def optimize(
+    dag: 'dag_lib.Dag',
+    minimize: common.OptimizeTarget = common.OptimizeTarget.COST,
+    blocked_resources: Optional[List['resources_lib.Resources']] = None,
+    quiet: bool = False,
+    request_options: Optional[admin_policy.RequestOptions] = None
+) -> 'dag_lib.Dag':
+    """Finds the best execution plan for the given DAG.
+
+    Args:
+        dag: the DAG to optimize.
+        minimize: whether to minimize cost or time.
+        blocked_resources: a list of resources that should not be used.
+        quiet: whether to suppress logging.
+        request_options: Request options used in enforcing admin policies.
+            This is only required when a admin policy is in use,
+            see: https://docs.skypilot.co/en/latest/cloud-setup/policy.html
+    Returns:
+        The optimized DAG.
+
+    Raises:
+        exceptions.ResourcesUnavailableError: if no resources are available
+            for a task.
+        exceptions.NoCloudAccessError: if no public clouds are enabled.
+    """
+    # TODO: We apply the admin policy only on the first DAG optimization which
+    # is shown on `sky launch`. The optimizer is also invoked during failover,
+    # but we do not apply the admin policy there. We should apply the admin
+    # policy in the optimizer, but that will require some refactoring.
+    dag, _ = admin_policy_utils.apply(
+        dag,
+        use_mutated_config_in_current_request=True,
+        request_options=request_options)
+    return optimizer.Optimizer.optimize(dag=dag,
+                                        minimize=minimize,
+                                        blocked_resources=blocked_resources,
+                                        quiet=quiet)
+
+
 @usage_lib.entrypoint
 def status(
     cluster_names: Optional[Union[str, List[str]]] = None,
@@ -325,8 +368,8 @@ def _start(
 
     usage_lib.record_cluster_name_for_current_operation(cluster_name)
 
-    with dag.Dag():
-        dummy_task = task.Task().set_resources(handle.launched_resources)
+    with dag_lib.Dag():
+        dummy_task = task_lib.Task().set_resources(handle.launched_resources)
         dummy_task.num_nodes = handle.launched_nodes
     handle = backend.provision(dummy_task,
                                to_provision=handle.launched_resources,
@@ -783,7 +826,7 @@ def cancel(
 def tail_logs(cluster_name: str,
               job_id: Optional[int],
               follow: bool = True,
-              tail: int = 0) -> None:
+              tail: int = 0) -> int:
     # NOTE(dev): Keep the docstring consistent between the Python API and CLI.
     """Tails the logs of a job.
 
@@ -799,6 +842,12 @@ def tail_logs(cluster_name: str,
           not the same as the user who created the cluster.
         sky.exceptions.CloudUserIdentityError: if we fail to get the current
           user identity.
+
+    Returns:
+        Return code based on success or failure of the job. 0 if success,
+          100 if the job failed. Note: This is not the return code of the job
+          script.
+
     """
     # Check the status of the cluster.
     handle = backend_utils.check_cluster_available(
@@ -808,7 +857,7 @@ def tail_logs(cluster_name: str,
     backend = backend_utils.get_backend_from_handle(handle)
 
     usage_lib.record_cluster_name_for_current_operation(cluster_name)
-    backend.tail_logs(handle, job_id, follow=follow, tail=tail)
+    return backend.tail_logs(handle, job_id, follow=follow, tail=tail)
 
 
 @usage_lib.entrypoint

sky/exceptions.py

@@ -9,7 +9,9 @@ from typing import Any, Dict, List, Optional, Sequence
 from sky.utils import env_options
 
 if typing.TYPE_CHECKING:
+    from sky import jobs as managed_jobs
     from sky.backends import backend
+    from sky.skylet import job_lib
     from sky.utils import status_lib
 
 # Return code for keyboard interruption and SIGTSTP
@@ -236,7 +238,7 @@ class CommandError(SkyPilotExcludeArgsBaseException):
         else:
             if (len(command) > 100 and
                     not env_options.Options.SHOW_DEBUG_INFO.get()):
-                # Chunck the command to avoid overflow.
+                # Chunk the command to avoid overflow.
                 command = command[:100] + '...'
             message = (f'Command {command} failed with return code '
                        f'{returncode}.\n{error_msg}')
@@ -449,3 +451,80 @@ class ApiServerConnectionError(RuntimeError):
             f'Could not connect to SkyPilot API server at {server_url}. '
             f'Please ensure that the server is running. '
             f'Try: curl {server_url}/api/health')
+
+
+class JobExitCode(enum.IntEnum):
+    """Job exit code enum.
+
+    These codes are used as return codes for job-related operations and as
+    process exit codes to indicate job status.
+    """
+
+    SUCCEEDED = 0
+    """The job completed successfully"""
+
+    FAILED = 100
+    """The job failed (due to user code, setup, or driver failure)"""
+
+    NOT_FINISHED = 101
+    """The job has not finished yet"""
+
+    NOT_FOUND = 102
+    """The job was not found"""
+
+    CANCELLED = 103
+    """The job was cancelled by the user"""
+
+    @classmethod
+    def from_job_status(cls,
+                        status: Optional['job_lib.JobStatus']) -> 'JobExitCode':
+        """Convert a job status to an exit code."""
+        # Import here to avoid circular imports
+        # pylint: disable=import-outside-toplevel
+        from sky.skylet import job_lib
+
+        if status is None:
+            return cls.NOT_FOUND
+
+        if not status.is_terminal():
+            return cls.NOT_FINISHED
+
+        if status == job_lib.JobStatus.SUCCEEDED:
+            return cls.SUCCEEDED
+
+        if status == job_lib.JobStatus.CANCELLED:
+            return cls.CANCELLED
+
+        if status in job_lib.JobStatus.user_code_failure_states(
+        ) or status == job_lib.JobStatus.FAILED_DRIVER:
+            return cls.FAILED
+
+        # Should not hit this case, but included to avoid errors
+        return cls.FAILED
+
+    @classmethod
+    def from_managed_job_status(
+            cls,
+            status: Optional['managed_jobs.ManagedJobStatus']) -> 'JobExitCode':
+        """Convert a managed job status to an exit code."""
+        # Import here to avoid circular imports
+        # pylint: disable=import-outside-toplevel
+        from sky import jobs as managed_jobs
+
+        if status is None:
+            return cls.NOT_FOUND
+
+        if not status.is_terminal():
+            return cls.NOT_FINISHED
+
+        if status == managed_jobs.ManagedJobStatus.SUCCEEDED:
+            return cls.SUCCEEDED
+
+        if status == managed_jobs.ManagedJobStatus.CANCELLED:
+            return cls.CANCELLED
+
+        if status.is_failed():
+            return cls.FAILED
+
+        # Should not hit this case, but included to avoid errors
+        return cls.FAILED

sky/jobs/client/sdk.py

@@ -184,7 +184,7 @@ def tail_logs(name: Optional[str] = None,
               follow: bool = True,
               controller: bool = False,
               refresh: bool = False,
-              output_stream: Optional['io.TextIOBase'] = None) -> None:
+              output_stream: Optional['io.TextIOBase'] = None) -> int:
     """Tails logs of managed jobs.
 
     You can provide either a job name or a job ID to tail logs. If both are not
@@ -199,6 +199,11 @@ def tail_logs(name: Optional[str] = None,
         output_stream: The stream to write the logs to. If None, print to the
             console.
 
+    Returns:
+        Exit code based on success or failure of the job. 0 if success,
+        100 if the job failed. See exceptions.JobExitCode for possible exit
+        codes.
+
     Request Raises:
         ValueError: invalid arguments.
         sky.exceptions.ClusterNotUpError: the jobs controller is not up.
@@ -217,7 +222,7 @@ def tail_logs(name: Optional[str] = None,
         timeout=(5, None),
     )
     request_id = server_common.get_request_id(response)
-    sdk.stream_response(request_id, response, output_stream)
+    return sdk.stream_response(request_id, response, output_stream)
 
 
 @usage_lib.entrypoint

sky/jobs/constants.py

@@ -40,7 +40,7 @@ JOBS_CLUSTER_NAME_PREFIX_LENGTH = 25
 # The version of the lib files that jobs/utils use. Whenever there is an API
 # change for the jobs/utils, we need to bump this version and update
 # job.utils.ManagedJobCodeGen to handle the version update.
-MANAGED_JOBS_VERSION = 2
+MANAGED_JOBS_VERSION = 3
 
 # The command for setting up the jobs dashboard on the controller. It firstly
 # checks if the systemd services are available, and if not (e.g., Kubernetes

sky/jobs/server/core.py

@@ -460,12 +460,17 @@ def cancel(name: Optional[str] = None,
 
 @usage_lib.entrypoint
 def tail_logs(name: Optional[str], job_id: Optional[int], follow: bool,
-              controller: bool, refresh: bool) -> None:
+              controller: bool, refresh: bool) -> int:
     # NOTE(dev): Keep the docstring consistent between the Python API and CLI.
     """Tail logs of managed jobs.
 
     Please refer to sky.cli.job_logs for documentation.
 
+    Returns:
+        Exit code based on success or failure of the job. 0 if success,
+        100 if the job failed. See exceptions.JobExitCode for possible exit
+        codes.
+
     Raises:
         ValueError: invalid arguments.
         sky.exceptions.ClusterNotUpError: the jobs controller is not up.
@@ -494,11 +499,11 @@ def tail_logs(name: Optional[str], job_id: Optional[int], follow: bool,
     backend = backend_utils.get_backend_from_handle(handle)
     assert isinstance(backend, backends.CloudVmRayBackend), backend
 
-    backend.tail_managed_job_logs(handle,
-                                  job_id=job_id,
-                                  job_name=name,
-                                  follow=follow,
-                                  controller=controller)
+    return backend.tail_managed_job_logs(handle,
+                                         job_id=job_id,
+                                         job_name=name,
+                                         follow=follow,
+                                         controller=controller)
 
 
 def start_dashboard_forwarding(refresh: bool = False) -> Tuple[int, int]:

sky/jobs/utils.py

@@ -511,8 +511,14 @@ def cancel_job_by_name(job_name: str) -> str:
     return f'Job {job_name!r} is scheduled to be cancelled.'
 
 
-def stream_logs_by_id(job_id: int, follow: bool = True) -> str:
-    """Stream logs by job id."""
+def stream_logs_by_id(job_id: int, follow: bool = True) -> Tuple[str, int]:
+    """Stream logs by job id.
+
+    Returns:
+        A tuple containing the log message and an exit code based on success or
+        failure of the job. 0 if success, 100 if the job failed.
+        See exceptions.JobExitCode for possible exit codes.
+    """
 
     def should_keep_logging(status: managed_job_state.ManagedJobStatus) -> bool:
         # If we see CANCELLING, just exit - we could miss some job logs but the
@@ -547,13 +553,16 @@ def stream_logs_by_id(job_id: int, follow: bool = True) -> str:
                             start_streaming = True
                         if start_streaming:
                             print(line, end='', flush=True)
-                return ''
+                return '', exceptions.JobExitCode.from_managed_job_status(
+                    managed_job_status)
             return (f'{colorama.Fore.YELLOW}'
                     f'Job {job_id} is already in terminal state '
                     f'{managed_job_status.value}. For more details, run: '
                     f'sky jobs logs --controller {job_id}'
                     f'{colorama.Style.RESET_ALL}'
-                    f'{job_msg}')
+                    f'{job_msg}',
+                    exceptions.JobExitCode.from_managed_job_status(
+                        managed_job_status))
         backend = backends.CloudVmRayBackend()
         task_id, managed_job_status = (
             managed_job_state.get_latest_task_id_status(job_id))
@@ -604,11 +613,12 @@ def stream_logs_by_id(job_id: int, follow: bool = True) -> str:
                                            job_id=None,
                                            managed_job_id=job_id,
                                            follow=follow)
-            if returncode == 0:
-                # If the log tailing exit successfully (the real job can be
-                # SUCCEEDED or FAILED), we can safely break the loop. We use the
-                # status in job queue to show the information, as the
-                # ManagedJobStatus is not updated yet.
+            if returncode in [rc.value for rc in exceptions.JobExitCode]:
+                # If the log tailing exits with a known exit code we can safely
+                # break the loop because it indicates the tailing process
+                # succeeded (even though the real job can be SUCCEEDED or
+                # FAILED). We use the status in job queue to show the
+                # information, as the ManagedJobStatus is not updated yet.
                 job_statuses = backend.get_job_status(handle, stream_logs=False)
                 job_status = list(job_statuses.values())[0]
                 assert job_status is not None, 'No job found.'
@@ -728,18 +738,25 @@ def stream_logs_by_id(job_id: int, follow: bool = True) -> str:
     logger.info(
         ux_utils.finishing_message(f'Managed job finished: {job_id} '
                                    f'(status: {managed_job_status.value}).'))
-    return ''
+    return '', exceptions.JobExitCode.from_managed_job_status(
+        managed_job_status)
 
 
 def stream_logs(job_id: Optional[int],
                 job_name: Optional[str],
                 controller: bool = False,
-                follow: bool = True) -> str:
-    """Stream logs by job id or job name."""
+                follow: bool = True) -> Tuple[str, int]:
+    """Stream logs by job id or job name.
+
+    Returns:
+        A tuple containing the log message and the exit code based on success
+        or failure of the job. 0 if success, 100 if the job failed.
+        See exceptions.JobExitCode for possible exit codes.
+    """
     if job_id is None and job_name is None:
         job_id = managed_job_state.get_latest_job_id()
         if job_id is None:
-            return 'No managed job found.'
+            return 'No managed job found.', exceptions.JobExitCode.NOT_FOUND
 
     if controller:
         if job_id is None:
@@ -754,7 +771,8 @@ def stream_logs(job_id: Optional[int],
                 if job['job_name'] == job_name
             }
             if not managed_job_ids:
-                return f'No managed job found with name {job_name!r}.'
+                return (f'No managed job found with name {job_name!r}.',
+                        exceptions.JobExitCode.NOT_FOUND)
             if len(managed_job_ids) > 1:
                 job_ids_str = ', '.join(
                     str(job_id) for job_id in managed_job_ids)
@@ -776,7 +794,7 @@ def stream_logs(job_id: Optional[int],
             if not follow:
                 # Assume that the log file hasn't been written yet. Since we
                 # aren't following, just return.
-                return ''
+                return '', exceptions.JobExitCode.SUCCEEDED
 
             job_status = managed_job_state.get_status(job_id)
             if job_status is None:
@@ -787,7 +805,8 @@ def stream_logs(job_id: Optional[int],
                 # point, it never will be. This job may have been submitted
                 # using an old version that did not create the log file, so this
                 # is not considered an exceptional case.
-                return ''
+                return '', exceptions.JobExitCode.from_managed_job_status(
+                    job_status)
 
             time.sleep(log_lib.SKY_LOG_WAITING_GAP_SECONDS)
 
@@ -833,15 +852,17 @@ def stream_logs(job_id: Optional[int],
 
         if follow:
             return ux_utils.finishing_message(
-                f'Job finished (status: {job_status}).')
+                f'Job finished (status: {job_status}).'
+            ), exceptions.JobExitCode.from_managed_job_status(job_status)
 
-        return ''
+        return '', exceptions.JobExitCode.SUCCEEDED
 
     if job_id is None:
         assert job_name is not None
         job_ids = managed_job_state.get_nonterminal_job_ids_by_name(job_name)
         if not job_ids:
-            return f'No running managed job found with name {job_name!r}.'
+            return (f'No running managed job found with name {job_name!r}.',
+                    exceptions.JobExitCode.NOT_FOUND)
         if len(job_ids) > 1:
             raise ValueError(
                 f'Multiple running jobs found with name {job_name!r}.')
@@ -1167,6 +1188,7 @@ class ManagedJobCodeGen:
       >> codegen = ManagedJobCodeGen.show_jobs(...)
     """
     _PREFIX = textwrap.dedent("""\
+        import sys
         from sky.jobs import utils
         from sky.jobs import state as managed_job_state
         from sky.jobs import constants as managed_job_constants
@@ -1222,9 +1244,17 @@ class ManagedJobCodeGen:
                     follow: bool = True,
                     controller: bool = False) -> str:
         code = textwrap.dedent(f"""\
-        msg = utils.stream_logs({job_id!r}, {job_name!r},
+        result = utils.stream_logs(job_id={job_id!r}, job_name={job_name!r},
                                 follow={follow}, controller={controller})
-        print(msg, flush=True)
+        if managed_job_version < 3:
+            # Versions 2 and older did not return a retcode, so we just print
+            # the result.
+            # TODO: Remove compatibility before 0.12.0
+            print(result, flush=True)
+        else:
+            msg, retcode = result
+            print(msg, flush=True)
+            sys.exit(retcode)
         """)
         return cls._build(code)
 

sky/server/constants.py

@@ -3,7 +3,7 @@
 # API server version, whenever there is a change in API server that requires a
 # restart of the local API server or error out when the client does not match
 # the server version.
-API_VERSION = '2'
+API_VERSION = '3'
 
 # Prefix for API request names.
 REQUEST_NAME_PREFIX = 'sky.'

sky/server/requests/payloads.py

@@ -12,6 +12,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union
 
 import pydantic
 
+from sky import admin_policy
 from sky import serve
 from sky import sky_logging
 from sky import skypilot_config
@@ -113,15 +114,9 @@ class CheckBody(RequestBody):
     verbose: bool
 
 
-class ValidateBody(RequestBody):
-    """The request body for the validate endpoint."""
-    dag: str
-
-
-class OptimizeBody(RequestBody):
-    """The request body for the optimize endpoint."""
+class DagRequestBody(RequestBody):
+    """Request body base class for endpoints with a dag."""
     dag: str
-    minimize: common_lib.OptimizeTarget = common_lib.OptimizeTarget.COST
 
     def to_kwargs(self) -> Dict[str, Any]:
         # Import here to avoid requirement of the whole SkyPilot dependency on
@@ -139,6 +134,19 @@ class OptimizeBody(RequestBody):
         return kwargs
 
 
+class ValidateBody(DagRequestBody):
+    """The request body for the validate endpoint."""
+    dag: str
+    request_options: Optional[admin_policy.RequestOptions]
+
+
+class OptimizeBody(DagRequestBody):
+    """The request body for the optimize endpoint."""
+    dag: str
+    minimize: common_lib.OptimizeTarget = common_lib.OptimizeTarget.COST
+    request_options: Optional[admin_policy.RequestOptions]
+
+
 class LaunchBody(RequestBody):
     """The request body for the launch endpoint."""
     task: str

sky/server/server.py

@@ -27,7 +27,6 @@ from sky import core
 from sky import exceptions
 from sky import execution
 from sky import global_user_state
-from sky import optimizer
 from sky import sky_logging
 from sky.clouds import service_catalog
 from sky.data import storage_utils
@@ -42,6 +41,7 @@ from sky.server.requests import payloads
 from sky.server.requests import requests as requests_lib
 from sky.skylet import constants
 from sky.usage import usage_lib
+from sky.utils import admin_policy_utils
 from sky.utils import common as common_lib
 from sky.utils import common_utils
 from sky.utils import dag_utils
@@ -258,9 +258,22 @@ async def validate(validate_body: payloads.ValidateBody) -> None:
     """Validates the user's DAG."""
     # TODO(SKY-1035): validate if existing cluster satisfies the requested
     # resources, e.g. sky exec --gpus V100:8 existing-cluster-with-no-gpus
+
+    # TODO: Our current launch process is split into three calls:
+    # validate, optimize, and launch. This requires us to apply the admin policy
+    # in each step, which may be an expensive operation. We should consolidate
+    # these into a single call or have a TTL cache for (task, admin_policy)
+    # pairs.
     logger.debug(f'Validating tasks: {validate_body.dag}')
     try:
         dag = dag_utils.load_chain_dag_from_yaml_str(validate_body.dag)
+        # TODO: Admin policy may contain arbitrary code, which may be expensive
+        # to run and may block the server thread. However, moving it into the
+        # executor adds a ~150ms penalty on the local API server because of
+        # added RTTs. For now, we stick to doing the validation inline in the
+        # server thread.
+        dag, _ = admin_policy_utils.apply(
+            dag, request_options=validate_body.request_options)
         for task in dag.tasks:
             # Will validate workdir and file_mounts in the backend, as those
             # need to be validated after the files are uploaded to the SkyPilot
@@ -283,7 +296,7 @@ async def optimize(optimize_body: payloads.OptimizeBody,
         request_name='optimize',
         request_body=optimize_body,
         ignore_return_value=True,
-        func=optimizer.Optimizer.optimize,
+        func=core.optimize,
         schedule_type=requests_lib.ScheduleType.SHORT,
     )
 

sky/skylet/constants.py

@@ -93,7 +93,7 @@ SKYLET_VERSION = '12'
 # The version of the lib files that skylet/jobs use. Whenever there is an API
 # change for the job_lib or log_lib, we need to bump this version, so that the
 # user can be notified to update their SkyPilot version on the remote cluster.
-SKYLET_LIB_VERSION = 2
+SKYLET_LIB_VERSION = 3
 SKYLET_VERSION_FILE = '~/.sky/skylet_version'
 
 # `sky jobs dashboard`-related

sky/skylet/job_lib.py

@@ -938,7 +938,9 @@ class JobLibCodeGen:
     _PREFIX = [
         'import os',
         'import getpass',
-        'from sky.skylet import job_lib, log_lib, constants',
+        'import sys',
+        'from sky import exceptions',
+        'from sky.skylet import log_lib, job_lib, constants',
     ]
 
     @classmethod
@@ -1033,6 +1035,13 @@ class JobLibCodeGen:
             f'tail_log_kwargs = {{"job_id": job_id, "log_dir": log_dir, "managed_job_id": {managed_job_id!r}, "follow": {follow}}}',
             f'{_LINUX_NEW_LINE}if getattr(constants, "SKYLET_LIB_VERSION", 1) > 1: tail_log_kwargs["tail"] = {tail}',
             f'{_LINUX_NEW_LINE}log_lib.tail_logs(**tail_log_kwargs)',
+            # After tailing, check the job status and exit with appropriate code
+            'job_status = job_lib.get_status(job_id)',
+            # Backward compatibility for returning exit code: Skylet versions 2
+            # and older did not have JobExitCode, so we use 0 for those versions
+            # TODO: Remove this special handling after 0.10.0.
+            'exit_code = exceptions.JobExitCode.from_job_status(job_status) if getattr(constants, "SKYLET_LIB_VERSION", 1) > 2 else 0',
+            'sys.exit(exit_code)',
         ]
         return cls._build(code)
 

{skypilot_nightly-1.0.0.dev20250303.dist-info → skypilot_nightly-1.0.0.dev20250304.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: skypilot-nightly
-Version: 1.0.0.dev20250303
+Version: 1.0.0.dev20250304
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0
@@ -241,13 +241,13 @@ pip install "skypilot-nightly[kubernetes,aws,gcp,azure,oci,lambda,runpod,fluidst
 </p>
 
 
-## Getting Started
+## Getting started
 You can find our documentation [here](https://docs.skypilot.co/).
 - [Installation](https://docs.skypilot.co/en/latest/getting-started/installation.html)
 - [Quickstart](https://docs.skypilot.co/en/latest/getting-started/quickstart.html)
 - [CLI reference](https://docs.skypilot.co/en/latest/reference/cli.html)
 
-## SkyPilot in 1 Minute
+## SkyPilot in 1 minute
 
 A SkyPilot task specifies: resource requirements, data to be synced, setup commands, and the task commands.
 
@@ -299,59 +299,46 @@ SkyPilot then performs the heavy-lifting for you, including:
 </p>
 
 
-Refer to [Quickstart](https://docs.skypilot.co/en/latest/getting-started/quickstart.html) to get started with SkyPilot.
+See [Quickstart](https://docs.skypilot.co/en/latest/getting-started/quickstart.html) to get started with SkyPilot.
 
-## More Information
+## Runnable examples
+
+See [**SkyPilot examples**](https://docs.skypilot.co/en/docs-examples/examples/index.html) that cover: development, training, serving, LLM models, AI apps, and common frameworks.
+
+Latest featured examples:
+
+| Task | Examples |
+|----------|----------|
+| Training | [PyTorch](https://docs.skypilot.co/en/latest/getting-started/tutorial.html), [DeepSpeed](https://docs.skypilot.co/en/latest/examples/training/deepspeed.html), [Finetune Llama 3](https://docs.skypilot.co/en/latest/examples/training/llama-3_1-finetuning.html), [NeMo](https://docs.skypilot.co/en/latest/examples/training/nemo.html), [Ray](https://docs.skypilot.co/en/latest/examples/training/ray.html), [Unsloth](https://docs.skypilot.co/en/latest/examples/training/unsloth.html), [Jax/TPU](https://docs.skypilot.co/en/latest/examples/training/tpu.html) |
+| Serving | [vLLM](https://docs.skypilot.co/en/latest/examples/serving/vllm.html), [SGLang](https://docs.skypilot.co/en/latest/examples/serving/sglang.html), [Ollama](https://docs.skypilot.co/en/latest/examples/serving/ollama.html) |
+| Models | [DeepSeek-R1](https://docs.skypilot.co/en/latest/examples/models/deepseek-r1.html), [Llama 3](https://docs.skypilot.co/en/latest/examples/models/llama-3.html), [CodeLlama](https://docs.skypilot.co/en/latest/examples/models/codellama.html), [Qwen](https://docs.skypilot.co/en/latest/examples/models/qwen.html), [Mixtral](https://docs.skypilot.co/en/latest/examples/models/mixtral.html) |
+| AI apps | [RAG](https://docs.skypilot.co/en/latest/examples/applications/rag.html), [vector databases](https://docs.skypilot.co/en/latest/examples/applications/vector_database.html) (ChromaDB, CLIP) |
+| Common frameworks | [Airflow](https://docs.skypilot.co/en/latest/examples/frameworks/airflow.html), [Jupyter](https://docs.skypilot.co/en/latest/examples/frameworks/jupyter.html) |
+
+Source files and more examples can be found in [`llm/`](https://github.com/skypilot-org/skypilot/tree/master/llm) and [`examples/`](https://github.com/skypilot-org/skypilot/tree/master/examples).
+
+## More information
 To learn more, see [SkyPilot Overview](https://docs.skypilot.co/en/latest/overview.html), [SkyPilot docs](https://docs.skypilot.co/en/latest/), and [SkyPilot blog](https://blog.skypilot.co/).
 
-<!-- Keep this section in sync with index.rst in SkyPilot Docs -->
-Runnable examples:
-- [**AI Gallery**](https://docs.skypilot.co/en/latest/gallery/index.html)
-- LLMs on SkyPilot
-  - [DeepSeek-R1](./llm/deepseek-r1/)
-  - [DeepSeek-Janus](./llm/deepseek-janus/)
-  - [Llama 3.2: lightweight and vision models](./llm/llama-3_2/)
-  - [Pixtral](./llm/pixtral/)
-  - [Llama 3.1 finetuning](./llm/llama-3_1-finetuning/) and [serving](./llm/llama-3_1/)
-  - [GPT-2 via `llm.c`](./llm/gpt-2/)
-  - [Llama 3](./llm/llama-3/)
-  - [Qwen](./llm/qwen/)
-  - [Databricks DBRX](./llm/dbrx/)
-  - [Gemma](./llm/gemma/)
-  - [Mixtral 8x7B](./llm/mixtral/); [Mistral 7B](https://docs.mistral.ai/self-deployment/skypilot/) (from official Mistral team)
-  - [Code Llama](./llm/codellama/)
-  - [vLLM: Serving LLM 24x Faster On the Cloud](./llm/vllm/) (from official vLLM team)
-  - [SGLang: Fast and Expressive LLM Serving On the Cloud](./llm/sglang/) (from official SGLang team)
-  - [Vicuna chatbots: Training & Serving](./llm/vicuna/) (from official Vicuna team)
-  - [Train your own Vicuna on Llama-2](./llm/vicuna-llama-2/)
-  - [Self-Hosted Llama-2 Chatbot](./llm/llama-2/)
-  - [Ollama: Quantized LLMs on CPUs](./llm/ollama/)
-  - [LoRAX](./llm/lorax/)
-  - [QLoRA](https://github.com/artidoro/qlora/pull/132)
-  - [LLaMA-LoRA-Tuner](https://github.com/zetavg/LLaMA-LoRA-Tuner#run-on-a-cloud-service-via-skypilot)
-  - [Tabby: Self-hosted AI coding assistant](https://github.com/TabbyML/tabby/blob/bed723fcedb44a6b867ce22a7b1f03d2f3531c1e/experimental/eval/skypilot.yaml)
-  - [LocalGPT](./llm/localgpt)
-  - [Falcon](./llm/falcon)
-  - Add yours here & see more in [`llm/`](./llm)!
-- Framework examples: [Vector Database](./examples/vector_database/), [PyTorch DDP](https://github.com/skypilot-org/skypilot/blob/master/examples/resnet_distributed_torch.yaml), [DeepSpeed](./examples/deepspeed-multinode/sky.yaml), [JAX/Flax on TPU](https://github.com/skypilot-org/skypilot/blob/master/examples/tpu/tpuvm_mnist.yaml), [Stable Diffusion](https://github.com/skypilot-org/skypilot/tree/master/examples/stable_diffusion), [Detectron2](https://github.com/skypilot-org/skypilot/blob/master/examples/detectron2_docker.yaml), [Distributed](https://github.com/skypilot-org/skypilot/blob/master/examples/resnet_distributed_tf_app.py) [TensorFlow](https://github.com/skypilot-org/skypilot/blob/master/examples/resnet_app_storage.yaml), [Ray Train](examples/distributed_ray_train/ray_train.yaml), [NeMo](https://github.com/skypilot-org/skypilot/blob/master/examples/nemo/), [programmatic grid search](https://github.com/skypilot-org/skypilot/blob/master/examples/huggingface_glue_imdb_grid_search_app.py), [Docker](https://github.com/skypilot-org/skypilot/blob/master/examples/docker/echo_app.yaml), [Cog](https://github.com/skypilot-org/skypilot/blob/master/examples/cog/), [Unsloth](https://github.com/skypilot-org/skypilot/blob/master/examples/unsloth/unsloth.yaml), [Ollama](https://github.com/skypilot-org/skypilot/blob/master/llm/ollama), [llm.c](https://github.com/skypilot-org/skypilot/tree/master/llm/gpt-2), [Airflow](./examples/airflow/training_workflow) and [many more (`examples/`)](./examples).
-
-Case Studies and Integrations: [Community Spotlights](https://blog.skypilot.co/community/)
+Case studies and integrations: [Community Spotlights](https://blog.skypilot.co/community/)
 
 Follow updates:
-- [Twitter](https://twitter.com/skypilot_org)
 - [Slack](http://slack.skypilot.co)
+- [X / Twitter](https://twitter.com/skypilot_org)
+- [LinkedIn](https://www.linkedin.com/company/skypilot-oss/)
 - [SkyPilot Blog](https://blog.skypilot.co/) ([Introductory blog post](https://blog.skypilot.co/introducing-skypilot/))
 
 Read the research:
 - [SkyPilot paper](https://www.usenix.org/system/files/nsdi23-yang-zongheng.pdf) and [talk](https://www.usenix.org/conference/nsdi23/presentation/yang-zongheng) (NSDI 2023)
 - [Sky Computing whitepaper](https://arxiv.org/abs/2205.07147)
 - [Sky Computing vision paper](https://sigops.org/s/conferences/hotos/2021/papers/hotos21-s02-stoica.pdf) (HotOS 2021)
-- [Policy for Managed Spot Jobs](https://www.usenix.org/conference/nsdi24/presentation/wu-zhanghao)  (NSDI 2024)
+- [SkyServe: AI serving across regions and clouds](https://arxiv.org/pdf/2411.01438) (EuroSys 2025)
+- [Managed jobs spot instance policy](https://www.usenix.org/conference/nsdi24/presentation/wu-zhanghao)  (NSDI 2024)
 
 SkyPilot was initially started at the [Sky Computing Lab](https://sky.cs.berkeley.edu) at UC Berkeley and has since gained many industry contributors. To read about the project's origin and vision, see [Concept: Sky Computing](https://docs.skypilot.co/en/latest/sky-computing.html).
 
-## Support and Questions
-We are excited to hear your feedback!
+## Questions and feedback
+We are excited to hear your feedback:
 * For issues and feature requests, please [open a GitHub issue](https://github.com/skypilot-org/skypilot/issues/new).
 * For questions, please use [GitHub Discussions](https://github.com/skypilot-org/skypilot/discussions).
 

{skypilot_nightly-1.0.0.dev20250303.dist-info → skypilot_nightly-1.0.0.dev20250304.dist-info}/RECORD

@@ -1,12 +1,12 @@
-sky/__init__.py,sha256=ylk3HVpmIoaOBWzfyeAB5oaee9NOryycO723C7F9ljs,6428
+sky/__init__.py,sha256=nBS4cQvvG4s3GLt8Ueva04qVaT4Zdxdxpq8oD7kS2SI,6428
 sky/admin_policy.py,sha256=hPo02f_A32gCqhUueF0QYy1fMSSKqRwYEg_9FxScN_s,3248
 sky/authentication.py,sha256=hCEqi77nprQEg3ktfRL51xiiw16zwZOmFEDB_Z7fWVU,22384
 sky/check.py,sha256=NDKx_Zm7YRxPjMv82wz3ESLnGIPljaACyqVdVNM0PzY,11258
-sky/cli.py,sha256=-3dv9uu0jp3uRgZYsdI_1SQTE6o0CD1XBql6gMydgHE,220651
+sky/cli.py,sha256=FDCSA5L__Djdk2Dc5eAmbE0YsfEu9zdE8vSu51oA19Q,221469
 sky/cloud_stores.py,sha256=kEHXd2divyra-1c3EusHxKyM5yTQlTXc6cKVXofsefA,23978
-sky/core.py,sha256=fDCit8n9BAXxJ9Dexq_CCOTmY8uZGK73wEaouNCRAbA,45573
+sky/core.py,sha256=MU9hcTdh8baMGrr2ZXmbxx12vNlhajrkeyg5QtV717c,47609
 sky/dag.py,sha256=Yl7Ry26Vql5cv4YMz8g9kOUgtoCihJnw7c8NgZYakMY,3242
-sky/exceptions.py,sha256=RCAojf2r9sUOdFYEhOoB05G-VP0xwkEVfsYdh2TZypA,13783
+sky/exceptions.py,sha256=MrDCfAmxmkTRSrUhOTLNNAYqEZLus_aErJm8b9SvbLk,16077
 sky/execution.py,sha256=0M4RTEzWn-B9oz221XdZOIGH12XOACmNq0j-WGUT_No,28023
 sky/global_user_state.py,sha256=sUDdSsJeiJkbgmZNwy8YGFK0XeNh-RBr1VDUvbmjf0g,33246
 sky/models.py,sha256=4xSW05BdDPEjW8Ubvj3VlVOVnzv0TbrolsFvR5R5v1U,638
@@ -34,7 +34,7 @@ sky/adaptors/vsphere.py,sha256=zJP9SeObEoLrpgHW2VHvZE48EhgVf8GfAEIwBeaDMfM,2129
 sky/backends/__init__.py,sha256=UDjwbUgpTRApbPJnNfR786GadUuwgRk3vsWoVu5RB_c,536
 sky/backends/backend.py,sha256=4BOqKZ-bwBTpjNnZF4JAHX2m2Iga7EmEn8Ao3tEivaM,7527
 sky/backends/backend_utils.py,sha256=B_46tG9PyrppxLWdg4mWGuuIr3TEcWTz6qhYXjAY2bw,133452
-sky/backends/cloud_vm_ray_backend.py,sha256=B8sH-m5pujvkTN_sLENsHw-SWpbaWwlHda26yI_1wnk,247459
+sky/backends/cloud_vm_ray_backend.py,sha256=KIU4IkUTBGE__7MC3ayjYMwE14mSxeiHjrGnK7wAQXw,247773
 sky/backends/docker_utils.py,sha256=Hyw1YY20EyghhEbYx6O2FIMDcGkNzBzV9TM7LFynei8,8358
 sky/backends/local_docker_backend.py,sha256=nSYCjms3HOPjPNOrcCqsUKm1WV3AAovRFjEQ7hcEXW4,17021
 sky/backends/wheel_utils.py,sha256=5BUzBqfYz7p1ME6_0PXGmcsAkLVb8NrFt317p7a4X8s,8278
@@ -43,9 +43,9 @@ sky/benchmark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/benchmark/benchmark_state.py,sha256=X8CXmuU9KgsDRhKedhFgjeRMUFWtQsjFs1qECvPG2yg,8723
 sky/benchmark/benchmark_utils.py,sha256=o4RymqSceq5mLEZL0upQM6NVEzJJQzj9s9tTm49uUTc,26365
 sky/client/__init__.py,sha256=pz6xvVSd9X-gwqbsDL0E9QOojYqM0KAD0j-NCyCIF1k,38
-sky/client/cli.py,sha256=-3dv9uu0jp3uRgZYsdI_1SQTE6o0CD1XBql6gMydgHE,220651
+sky/client/cli.py,sha256=FDCSA5L__Djdk2Dc5eAmbE0YsfEu9zdE8vSu51oA19Q,221469
 sky/client/common.py,sha256=axDic7WOG1e78SdFm5XIwdhX7YNvf3g4k7INrsW3X4s,14611
-sky/client/sdk.py,sha256=zNjZFDTecloxdkGqmSq9wrdKjX9pEMJLghu5bUThJWk,67406
+sky/client/sdk.py,sha256=IRx72BXqOn_WVvtOuTXfgR5zcSm_lyoXeYxa5c_2_qk,68723
 sky/clouds/__init__.py,sha256=OW6mJ-9hpJSBORCgt2LippLQEYZHNfnBW1mooRNNvxo,1416
 sky/clouds/aws.py,sha256=J8tczaTDL239UowN9tUlhI92SeHw01wtFucSckvG63w,54112
 sky/clouds/azure.py,sha256=bawEw6wOLAVyrjxMD-4UjLCuMj1H5_jH8qggpfZYS54,31703
@@ -107,19 +107,19 @@ sky/data/mounting_utils.py,sha256=i79Y-DZXVR88fjG_MBPB8EgsZBnHdpf1LGnJSm_VhAg,16
 sky/data/storage.py,sha256=mTgMGdfSV6Gia076Dvgmc18ZlqF6eObima558UShiXA,207165
 sky/data/storage_utils.py,sha256=zB99nRTJjh8isU0UmqERmlwwRNgfig91IwrwVH8CcNw,12383
 sky/jobs/__init__.py,sha256=qoI53-xXE0-SOkrLWigvhgFXjk7dWE0OTqGPYIk-kmM,1458
-sky/jobs/constants.py,sha256=Yv4cIUhkM-kIMlwcmhiFVPXI7aLnLw7MBhStDqSjjOM,3088
+sky/jobs/constants.py,sha256=1XiIqdR5dEgGgepLKWkZCRT3MYSsMBR-dO7N4RTsjwg,3088
 sky/jobs/controller.py,sha256=4G1CKI7M7D1BgJLbJMeqzg0iDDv7FR4ObB1BKZFFjhk,29585
 sky/jobs/recovery_strategy.py,sha256=RLrqq8B1likxTknPzt3_BqO26sFVpoatxzUuGfwc18A,26170
 sky/jobs/scheduler.py,sha256=8k2ieJ1TTvJ0TOalnklJtrMwFuatsh-ojoPMBgFRBlI,13119
 sky/jobs/state.py,sha256=tDULLH6DVs4oKUIKhh0UAn3RzyVGuIUtEq5kW7K1Ojw,44585
-sky/jobs/utils.py,sha256=Ta0UJXNhQWaNdKHPw4ZIF_o82m_AUH2RKE2Uj4U_TKM,54235
+sky/jobs/utils.py,sha256=O1cOXeWXzZNxQzEZ4xwadskQr1Azm1pCRe4Ju0dfvfg,55845
 sky/jobs/client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sky/jobs/client/sdk.py,sha256=RBaU4IR9e8oMMzpMa-yMlpcd3sD5M4_c5xbp2VPbz9U,9939
+sky/jobs/client/sdk.py,sha256=4STtriCWLUq1mm-tEsh_iXC7r-U7_PY0R9X6-DNpaXs,10122
 sky/jobs/dashboard/dashboard.py,sha256=JaVrNUEFQPLmsDZnrR76Uo8QqcAHdgYzx7GZTxDfl9M,7885
 sky/jobs/dashboard/static/favicon.ico,sha256=uYlvgxSM7gjBmXpZ8wydvZUPAbJiiix-rc2Xe5mma9s,15086
 sky/jobs/dashboard/templates/index.html,sha256=tz95q8O2pF7IvfY6yv0rnPyhj4DX8WX4RIVVxqFKV1Y,28519
 sky/jobs/server/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-sky/jobs/server/core.py,sha256=K5qxukyBnPD1DjcNMFJheULWoYSLaVXtjt0D1Cb9GoI,24902
+sky/jobs/server/core.py,sha256=s6A3KJsSQz1GlD6qfJ-XiEg6scc3sqMTqVd1Kr6ZTIU,25113
 sky/jobs/server/dashboard_utils.py,sha256=2Mbx40W1pQqPEPHsSDbHeaF0j5cgyKy-_A9Owdwp_AQ,2315
 sky/jobs/server/server.py,sha256=vdVxl4ZkBRlfOdsUO5Ttxon_-NE9XoMVMSo8fJ-Y73Y,7803
 sky/provision/__init__.py,sha256=LzOo5LjkRXwSf29dUqN14YbjzQu3liXLQcmweTeZ4dE,6457
@@ -229,13 +229,13 @@ sky/serve/server/core.py,sha256=pRvFadEIH_WTUkTtSmuFoPBP4JFq8Obt68ifi9DWuog,3686
 sky/serve/server/server.py,sha256=gQGVU9nHYdGbaLhGjIUNIYn4xwKjRASRJkiiTL5AI1Y,3283
 sky/server/__init__.py,sha256=MPPBqFzXz6Jv5QSk6td_IcvnfXfNErDZVcizu4MLRow,27
 sky/server/common.py,sha256=pEa-q3P5aOm6RMlit0pVzlDoJnZU_6zViO7aK_7htn0,17843
-sky/server/constants.py,sha256=89jKE3SIe1T3_7j6ECTy4pZnhZZD7fBwsWOCOkTban8,770
-sky/server/server.py,sha256=IDVx6qHl75PRSyc23NLMi0JLjqrTjOHWiCeNvK1D60U,42511
+sky/server/constants.py,sha256=_ZNrxYh8vmgbf3DmkGDduxjvO2y43ZSPTkH5rCNsVjU,770
+sky/server/server.py,sha256=VOro33c4ZybLeZF57ANiZRWUjtyUvCDEeQMX7yd_HYE,43271
 sky/server/stream_utils.py,sha256=-3IX1YCgxAFfcvQIV0TCvOn1wbRLWovAx3ckCrsExWU,5651
 sky/server/html/log.html,sha256=TSGZktua9Ysl_ysg3w60rjxAxhH61AJnsYDHdtqrjmI,6929
 sky/server/requests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/server/requests/executor.py,sha256=Jk8RJoQlicDqaHhgVWMH3UiL-dJS7lGSGd05GPv-Lrc,19781
-sky/server/requests/payloads.py,sha256=75bHfIJ5TR8z3BVVKMYLKxK5BzjtxwawCpwN8B88-FA,16077
+sky/server/requests/payloads.py,sha256=nVb7vr1SNAq6ay2dNe9301zLHp7NrM79M7nsWAECBms,16340
 sky/server/requests/requests.py,sha256=aMdjiK5kjSYP36pxdXFU6qgKOXcOmtViHbFm3V8Dvf8,19590
 sky/server/requests/queues/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/server/requests/queues/mp_queue.py,sha256=_7AFas__0b1L8e7Bwy4lu0VYU18R85YwMlDHPhQCfh0,2998
@@ -250,9 +250,9 @@ sky/skylet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/skylet/attempt_skylet.py,sha256=GZ6ITjjA0m-da3IxXXfoHR6n4pjp3X3TOXUqVvSrV0k,2136
 sky/skylet/autostop_lib.py,sha256=W4CtMira6QnmYToFT5kYTGjNPRZNC-bZPfsF1k3tluE,4480
 sky/skylet/configs.py,sha256=UtnpmEL0F9hH6PSjhsps7xgjGZ6qzPOfW1p2yj9tSng,1887
-sky/skylet/constants.py,sha256=wELTCOQLIafwtl7_L8vjDwdqYQQxG-Sup3H7vxmcyB4,18037
+sky/skylet/constants.py,sha256=6I_JGIQXHGDNodNCQYLqflmJotDVnFUt4R48BASyUN0,18037
 sky/skylet/events.py,sha256=pnV3ZiwWhXqTHpU5B5Y9Xwam_7FQDI6IrxgSx7X_NVA,12743
-sky/skylet/job_lib.py,sha256=8W6GM2zxqGMIoD3AGiXcKsK_7-qouuTojiVL6upSeoA,43728
+sky/skylet/job_lib.py,sha256=j_VRDWcEGIStLLEC0cD9B3JxggPJOZaDAaNKe50uhy4,44319
 sky/skylet/log_lib.py,sha256=DzOrgY8C7RdEMLC9O9kEKV-iLMb9wVMPSnDha8eMx28,20900
 sky/skylet/log_lib.pyi,sha256=rRk4eUX0RHGs1QL9CXsJq6RE7FqqxZlfuPJOLXTvg7I,4453
 sky/skylet/skylet.py,sha256=mWmqCvxSlfdVU_L8NL6P52jmCt3smd8K0HdyNBfMPeI,1234
@@ -344,9 +344,9 @@ sky/utils/kubernetes/k8s_gpu_labeler_setup.yaml,sha256=VLKT2KKimZu1GDg_4AIlIt488
 sky/utils/kubernetes/kubernetes_deploy_utils.py,sha256=otzHzpliHDCpzYT-nU9Q0ZExbiFpDPWvhxwkvchZj7k,10073
 sky/utils/kubernetes/rsync_helper.sh,sha256=h4YwrPFf9727CACnMJvF3EyK_0OeOYKKt4su_daKekw,1256
 sky/utils/kubernetes/ssh_jump_lifecycle_manager.py,sha256=Kq1MDygF2IxFmu9FXpCxqucXLmeUrvs6OtRij6XTQbo,6554
-skypilot_nightly-1.0.0.dev20250303.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
-skypilot_nightly-1.0.0.dev20250303.dist-info/METADATA,sha256=VVOYmK48D7iagTqlQiCjx2Qc6nk11wHrI1SDcHIRTkE,19115
-skypilot_nightly-1.0.0.dev20250303.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-skypilot_nightly-1.0.0.dev20250303.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
-skypilot_nightly-1.0.0.dev20250303.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
-skypilot_nightly-1.0.0.dev20250303.dist-info/RECORD,,
+skypilot_nightly-1.0.0.dev20250304.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
+skypilot_nightly-1.0.0.dev20250304.dist-info/METADATA,sha256=jiLF-ux0ZwUp4eKySgZB9QR7PrPU24Iavg7gSuzVKcw,18173
+skypilot_nightly-1.0.0.dev20250304.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+skypilot_nightly-1.0.0.dev20250304.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
+skypilot_nightly-1.0.0.dev20250304.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
+skypilot_nightly-1.0.0.dev20250304.dist-info/RECORD,,