skypilot-nightly 1.0.0.dev20250905__py3-none-any.whl → 1.0.0.dev20251203__py3-none-any.whl

sky/jobs/__init__.py

@@ -11,6 +11,7 @@ from sky.jobs.client.sdk import pool_status
 from sky.jobs.client.sdk import pool_sync_down_logs
 from sky.jobs.client.sdk import pool_tail_logs
 from sky.jobs.client.sdk import queue
+from sky.jobs.client.sdk import queue_v2
 from sky.jobs.client.sdk import tail_logs
 from sky.jobs.constants import JOBS_CLUSTER_NAME_PREFIX_LENGTH
 from sky.jobs.constants import JOBS_CONTROLLER_LOGS_DIR
@@ -38,6 +39,7 @@ __all__ = [
     'cancel',
     'launch',
     'queue',
+    'queue_v2',
     'tail_logs',
     'dashboard',
     'download_logs',

sky/jobs/client/sdk.py

@@ -9,11 +9,13 @@ from sky import sky_logging
 from sky.adaptors import common as adaptors_common
 from sky.client import common as client_common
 from sky.client import sdk
+from sky.schemas.api import responses
 from sky.serve.client import impl
 from sky.server import common as server_common
 from sky.server import rest
 from sky.server import versions
 from sky.server.requests import payloads
+from sky.server.requests import request_names
 from sky.skylet import constants
 from sky.usage import usage_lib
 from sky.utils import admin_policy_utils
@@ -82,8 +84,11 @@ def launch(
         raise click.UsageError('Cannot specify num_jobs without pool.')
 
     dag = dag_utils.convert_entrypoint_to_dag(task)
+
     with admin_policy_utils.apply_and_use_config_in_current_request(
-            dag, at_client_side=True) as dag:
+            dag,
+            request_name=request_names.AdminPolicyRequestName.JOBS_LAUNCH,
+            at_client_side=True) as dag:
         sdk.validate(dag)
         if _need_confirmation:
             job_identity = 'a managed job'
@@ -123,6 +128,87 @@ def launch(
         return server_common.get_request_id(response)
 
 
+@usage_lib.entrypoint
+@server_common.check_server_healthy_or_start
+@versions.minimal_api_version(18)
+def queue_v2(
+    refresh: bool,
+    skip_finished: bool = False,
+    all_users: bool = False,
+    job_ids: Optional[List[int]] = None,
+    limit: Optional[int] = None,
+    fields: Optional[List[str]] = None,
+) -> server_common.RequestId[Tuple[List[responses.ManagedJobRecord], int, Dict[
+        str, int], int]]:
+    """Gets statuses of managed jobs.
+
+    Please refer to sky.cli.job_queue for documentation.
+
+    Args:
+        refresh: Whether to restart the jobs controller if it is stopped.
+        skip_finished: Whether to skip finished jobs.
+        all_users: Whether to show all users' jobs.
+        job_ids: IDs of the managed jobs to show.
+        limit: Number of jobs to show.
+        fields: Fields to get for the managed jobs.
+
+    Returns:
+        The request ID of the queue request.
+
+    Request Returns:
+        job_records (List[responses.ManagedJobRecord]): A list of dicts, with each dict
+          containing the information of a job.
+
+          .. code-block:: python
+
+            [
+              {
+                'job_id': (int) job id,
+                'job_name': (str) job name,
+                'resources': (str) resources of the job,
+                'submitted_at': (float) timestamp of submission,
+                'end_at': (float) timestamp of end,
+                'job_duration': (float) duration in seconds,
+                'recovery_count': (int) Number of retries,
+                'status': (sky.jobs.ManagedJobStatus) of the job,
+                'cluster_resources': (str) resources of the cluster,
+                'region': (str) region of the cluster,
+                'task_id': (int), set to 0 (except in pipelines, which may have multiple tasks), # pylint: disable=line-too-long
+                'task_name': (str), same as job_name (except in pipelines, which may have multiple tasks), # pylint: disable=line-too-long
+              }
+            ]
+        total (int): Total number of jobs after filter,
+        status_counts (Dict[str, int]): Status counts after filter,
+        total_no_filter (int): Total number of jobs before filter,
+
+    Request Raises:
+        sky.exceptions.ClusterNotUpError: the jobs controller is not up or
+          does not exist.
+        RuntimeError: if failed to get the managed jobs with ssh.
+    """
+    body = payloads.JobsQueueV2Body(
+        refresh=refresh,
+        skip_finished=skip_finished,
+        all_users=all_users,
+        job_ids=job_ids,
+        limit=limit,
+        fields=fields,
+    )
+    path = '/jobs/queue/v2'
+    response = server_common.make_authenticated_request(
+        'POST',
+        path,
+        json=json.loads(body.model_dump_json()),
+        timeout=(5, None))
+    return server_common.get_request_id(response=response)
+
+
+# Deprecated. Please use queue_v2 instead for better performance.
+# In https://github.com/skypilot-org/skypilot/pull/7695, the `queue` function
+# is updated to return new typed data for performance improvement if the API
+# server supports it, which breaks the backward compatibility.
+# In https://github.com/skypilot-org/skypilot/pull/8015, we revert the change
+# and add a new function `queue_v2` to return the new typed data.
 @usage_lib.entrypoint
 @server_common.check_server_healthy_or_start
 def queue(
@@ -130,9 +216,11 @@ def queue(
     skip_finished: bool = False,
     all_users: bool = False,
     job_ids: Optional[List[int]] = None
-) -> server_common.RequestId[List[Dict[str, Any]]]:
+) -> server_common.RequestId[List[responses.ManagedJobRecord]]:
     """Gets statuses of managed jobs.
 
+    Deprecated. Please use queue_v2 instead for better performance.
+
     Please refer to sky.cli.job_queue for documentation.
 
     Args:
@@ -145,7 +233,7 @@ def queue(
         The request ID of the queue request.
 
     Request Returns:
-        job_records (List[Dict[str, Any]]): A list of dicts, with each dict
+        job_records (List[responses.ManagedJobRecord]): A list of dicts, with each dict
           containing the information of a job.
 
           .. code-block:: python
@@ -383,15 +471,24 @@ def dashboard() -> None:
 @server_common.check_server_healthy_or_start
 @versions.minimal_api_version(12)
 def pool_apply(
-    task: Union['sky.Task', 'sky.Dag'],
+    task: Optional[Union['sky.Task', 'sky.Dag']],
     pool_name: str,
     mode: 'serve_utils.UpdateMode',
+    workers: Optional[int] = None,
     # Internal only:
     # pylint: disable=invalid-name
     _need_confirmation: bool = False
 ) -> server_common.RequestId[None]:
     """Apply a config to a pool."""
+    remote_api_version = versions.get_remote_api_version()
+    if (workers is not None and
+        (remote_api_version is None or remote_api_version < 19)):
+        raise click.UsageError('Updating the number of workers in a pool is '
+                               'not supported in your API server. Please '
+                               'upgrade to a newer API server to use this '
+                               'feature.')
     return impl.apply(task,
+                      workers,
                       pool_name,
                       mode,
                       pool=True,

sky/jobs/client/sdk_async.py

@@ -1,12 +1,13 @@
 """Async SDK functions for managed jobs."""
 import typing
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Dict, List, Optional, Tuple, Union
 
 from sky import backends
 from sky import sky_logging
 from sky.adaptors import common as adaptors_common
 from sky.client import sdk_async
 from sky.jobs.client import sdk
+from sky.schemas.api import responses
 from sky.skylet import constants
 from sky.usage import usage_lib
 from sky.utils import common_utils
@@ -28,6 +29,8 @@ logger = sky_logging.init_logger(__name__)
 async def launch(
     task: Union['sky.Task', 'sky.Dag'],
     name: Optional[str] = None,
+    pool: Optional[str] = None,
+    num_jobs: Optional[int] = None,
     # Internal only:
     # pylint: disable=invalid-name
     _need_confirmation: bool = False,
@@ -35,8 +38,29 @@ async def launch(
         sdk_async.StreamConfig] = sdk_async.DEFAULT_STREAM_CONFIG,
 ) -> Tuple[Optional[int], Optional[backends.ResourceHandle]]:
     """Async version of launch() that launches a managed job."""
-    request_id = await context_utils.to_thread(sdk.launch, task, name,
-                                               _need_confirmation)
+    request_id = await context_utils.to_thread(sdk.launch, task, name, pool,
+                                               num_jobs, _need_confirmation)
+    if stream_logs is not None:
+        return await sdk_async._stream_and_get(request_id, stream_logs)  # pylint: disable=protected-access
+    else:
+        return await sdk_async.get(request_id)
+
+
+@usage_lib.entrypoint
+async def queue_v2(
+    refresh: bool,
+    skip_finished: bool = False,
+    all_users: bool = False,
+    job_ids: Optional[List[int]] = None,
+    limit: Optional[int] = None,
+    fields: Optional[List[str]] = None,
+    stream_logs: Optional[
+        sdk_async.StreamConfig] = sdk_async.DEFAULT_STREAM_CONFIG
+) -> Tuple[List[responses.ManagedJobRecord], int, Dict[str, int], int]:
+    """Async version of queue_v2() that gets statuses of managed jobs."""
+    request_id = await context_utils.to_thread(sdk.queue_v2, refresh,
+                                               skip_finished, all_users,
+                                               job_ids, limit, fields)
     if stream_logs is not None:
         return await sdk_async._stream_and_get(request_id, stream_logs)  # pylint: disable=protected-access
     else:
@@ -48,12 +72,14 @@ async def queue(
     refresh: bool,
     skip_finished: bool = False,
     all_users: bool = False,
+    job_ids: Optional[List[int]] = None,
     stream_logs: Optional[
         sdk_async.StreamConfig] = sdk_async.DEFAULT_STREAM_CONFIG
-) -> List[Dict[str, Any]]:
+) -> List[responses.ManagedJobRecord]:
     """Async version of queue() that gets statuses of managed jobs."""
     request_id = await context_utils.to_thread(sdk.queue, refresh,
-                                               skip_finished, all_users)
+                                               skip_finished, all_users,
+                                               job_ids)
     if stream_logs is not None:
         return await sdk_async._stream_and_get(request_id, stream_logs)  # pylint: disable=protected-access
     else:

sky/jobs/constants.py

@@ -1,4 +1,5 @@
 """Constants used for Managed Jobs."""
+import os
 from typing import Any, Dict, Union
 
 from sky.skylet import constants as skylet_constants
@@ -9,17 +10,21 @@ JOBS_CONTROLLER_LOGS_DIR = '~/sky_logs/jobs_controller'
 
 JOBS_TASK_YAML_PREFIX = '~/.sky/managed_jobs'
 
+JOB_CONTROLLER_INDICATOR_FILE = '~/.sky/is_jobs_controller'
+
+CONSOLIDATED_SIGNAL_PATH = os.path.expanduser('~/.sky/signals/')
+SIGNAL_FILE_PREFIX = '/tmp/sky_jobs_controller_signal_{}'
+
+# The consolidation mode lock ensures that if multiple API servers are running
+# at the same time (e.g. during a rolling update), recovery can only happen once
+# the previous API server has exited.
+CONSOLIDATION_MODE_LOCK_ID = '~/.sky/consolidation_mode_lock'
+
 # Resources as a dict for the jobs controller.
-# Use smaller CPU instance type for jobs controller, but with more memory, i.e.
-# r6i.xlarge (4vCPUs, 32 GB) for AWS, Standard_E4s_v5 (4vCPUs, 32 GB) for Azure,
-# and n2-highmem-4 (4 vCPUs, 32 GB) for GCP, etc.
-# Concurrently limits are set based on profiling. 4x num vCPUs is the launch
-# parallelism limit, and memory / 350MB is the limit to concurrently running
-# jobs. See _get_launch_parallelism and _get_job_parallelism in scheduler.py.
 # We use 50 GB disk size to reduce the cost.
 CONTROLLER_RESOURCES: Dict[str, Union[str, int]] = {
     'cpus': '4+',
-    'memory': '8x',
+    'memory': '4x',
     'disk_size': 50
 }
 
@@ -47,7 +52,9 @@ JOBS_CLUSTER_NAME_PREFIX_LENGTH = 25
 # The version of the lib files that jobs/utils use. Whenever there is an API
 # change for the jobs/utils, we need to bump this version and update
 # job.utils.ManagedJobCodeGen to handle the version update.
-MANAGED_JOBS_VERSION = 10
+# WARNING: If you update this due to a codegen change, make sure to make the
+# corresponding change in the ManagedJobsService AND bump the SKYLET_VERSION.
+MANAGED_JOBS_VERSION = 12
 
 # The command for setting up the jobs dashboard on the controller. It firstly
 # checks if the systemd services are available, and if not (e.g., Kubernetes