skypilot-nightly 1.0.0.dev20251009__py3-none-any.whl → 1.0.0.dev20251012__py3-none-any.whl

sky/server/requests/requests.py

@@ -292,6 +292,100 @@ class Request:
             raise
 
 
+def encode_requests(requests: List[Request]) -> List[payloads.RequestPayload]:
+    """Serialize the SkyPilot API request for display purposes.
+
+        This function should be called on the server side to serialize the
+        request body into human readable format, e.g., the entrypoint should
+        be a string, and the pid, error, or return value are not needed.
+
+        The returned value will then be displayed on the client side in request
+        table.
+
+        We do not use `encode` for display to avoid a large amount of data being
+        sent to the client side, especially for the request table could include
+        all the requests.
+        """
+    encoded_requests = []
+    all_users = global_user_state.get_all_users()
+    all_users_map = {user.id: user.name for user in all_users}
+    for request in requests:
+        if request.request_body is not None:
+            assert isinstance(request.request_body,
+                              payloads.RequestBody), (request.name,
+                                                      request.request_body)
+        user_name = all_users_map.get(request.user_id)
+        payload = payloads.RequestPayload(
+            request_id=request.request_id,
+            name=request.name,
+            entrypoint=request.entrypoint.__name__
+            if request.entrypoint is not None else '',
+            request_body=request.request_body.model_dump_json()
+            if request.request_body is not None else json.dumps(None),
+            status=request.status.value,
+            return_value=json.dumps(None),
+            error=json.dumps(None),
+            pid=None,
+            created_at=request.created_at,
+            schedule_type=request.schedule_type.value,
+            user_id=request.user_id,
+            user_name=user_name,
+            cluster_name=request.cluster_name,
+            status_msg=request.status_msg,
+            should_retry=request.should_retry,
+            finished_at=request.finished_at,
+        )
+        encoded_requests.append(payload)
+    return encoded_requests
+
+
+def _update_request_row_fields(
+        row: Tuple[Any, ...],
+        fields: Optional[List[str]] = None) -> Tuple[Any, ...]:
+    """Update the request row fields."""
+    if not fields:
+        return row
+
+    # Convert tuple to dictionary for easier manipulation
+    content = dict(zip(fields, row))
+
+    # Required fields in RequestPayload
+    if 'request_id' not in fields:
+        content['request_id'] = ''
+    if 'name' not in fields:
+        content['name'] = ''
+    if 'entrypoint' not in fields:
+        content['entrypoint'] = server_constants.EMPTY_PICKLED_VALUE
+    if 'request_body' not in fields:
+        content['request_body'] = server_constants.EMPTY_PICKLED_VALUE
+    if 'status' not in fields:
+        content['status'] = RequestStatus.PENDING.value
+    if 'created_at' not in fields:
+        content['created_at'] = 0
+    if 'user_id' not in fields:
+        content['user_id'] = ''
+    if 'return_value' not in fields:
+        content['return_value'] = json.dumps(None)
+    if 'error' not in fields:
+        content['error'] = json.dumps(None)
+    if 'schedule_type' not in fields:
+        content['schedule_type'] = ScheduleType.SHORT.value
+    # Optional fields in RequestPayload
+    if 'pid' not in fields:
+        content['pid'] = None
+    if 'cluster_name' not in fields:
+        content['cluster_name'] = None
+    if 'status_msg' not in fields:
+        content['status_msg'] = None
+    if 'should_retry' not in fields:
+        content['should_retry'] = False
+    if 'finished_at' not in fields:
+        content['finished_at'] = None
+
+    # Convert back to tuple in the same order as REQUEST_COLUMNS
+    return tuple(content[col] for col in REQUEST_COLUMNS)
+
+
 def kill_cluster_requests(cluster_name: str, exclude_request_name: str):
     """Kill all pending and running requests for a cluster.
 
@@ -634,6 +728,7 @@ class RequestTaskFilter:
             Mutually exclusive with exclude_request_names.
         finished_before: if provided, only include requests finished before this
             timestamp.
+        limit: the number of requests to show. If None, show all requests.
 
     Raises:
         ValueError: If both exclude_request_names and include_request_names are
@@ -645,6 +740,8 @@ class RequestTaskFilter:
     exclude_request_names: Optional[List[str]] = None
     include_request_names: Optional[List[str]] = None
     finished_before: Optional[float] = None
+    limit: Optional[int] = None
+    fields: Optional[List[str]] = None
 
     def __post_init__(self):
         if (self.exclude_request_names is not None and
@@ -687,8 +784,13 @@ class RequestTaskFilter:
         if filter_str:
             filter_str = f' WHERE {filter_str}'
         columns_str = ', '.join(REQUEST_COLUMNS)
-        return (f'SELECT {columns_str} FROM {REQUEST_TABLE}{filter_str} '
-                'ORDER BY created_at DESC'), filter_params
+        if self.fields:
+            columns_str = ', '.join(self.fields)
+        query_str = (f'SELECT {columns_str} FROM {REQUEST_TABLE}{filter_str} '
+                     'ORDER BY created_at DESC')
+        if self.limit is not None:
+            query_str += f' LIMIT {self.limit}'
+        return query_str, filter_params
 
 
 @init_db
@@ -722,6 +824,21 @@ async def get_request_tasks_async(
     return [Request.from_row(row) for row in rows]
 
 
+@init_db_async
+@metrics_lib.time_me_async
+async def get_request_tasks_with_fields_async(
+    req_filter: RequestTaskFilter,
+    fields: Optional[List[str]] = None,
+) -> List[Request]:
+    """Async version of get_request_tasks."""
+    assert _DB is not None
+    async with _DB.execute_fetchall_async(*req_filter.build_query()) as rows:
+        if not rows:
+            return []
+    rows = [_update_request_row_fields(row, fields) for row in rows]
+    return [Request.from_row(row) for row in rows]
+
+
 @init_db_async
 @metrics_lib.time_me_async
 async def get_api_request_ids_start_with(incomplete: str) -> List[str]:

sky/server/server.py

@@ -1243,7 +1243,7 @@ async def logs(
     background_tasks.add_task(task.cancel)
     # TODO(zhwu): This makes viewing logs in browser impossible. We should adopt
     # the same approach as /stream.
-    return stream_utils.stream_response(
+    return stream_utils.stream_response_for_long_request(
         request_id=request.state.request_id,
         logs_path=request_task.log_path,
         background_tasks=background_tasks,
@@ -1539,6 +1539,7 @@ async def stream(
                 'X-Accel-Buffering': 'no'
             })
 
+    polling_interval = stream_utils.DEFAULT_POLL_INTERVAL
     # Original plain text streaming logic
     if request_id is not None:
         request_task = await requests_lib.get_request_async(request_id)
@@ -1553,6 +1554,8 @@ async def stream(
             raise fastapi.HTTPException(
                 status_code=404,
                 detail=f'Log of request {request_id!r} has been deleted')
+        if request_task.schedule_type == requests_lib.ScheduleType.LONG:
+            polling_interval = stream_utils.LONG_REQUEST_POLL_INTERVAL
     else:
         assert log_path is not None, (request_id, log_path)
         if log_path == constants.API_SERVER_LOGS:
@@ -1600,7 +1603,8 @@ async def stream(
                                           log_path_to_stream,
                                           plain_logs=format == 'plain',
                                           tail=tail,
-                                          follow=follow),
+                                          follow=follow,
+                                          polling_interval=polling_interval),
         media_type='text/plain',
         headers=headers,
     )
@@ -1625,6 +1629,10 @@ async def api_status(
         None, description='Request IDs to get status for.'),
     all_status: bool = fastapi.Query(
         False, description='Get finished requests as well.'),
+    limit: Optional[int] = fastapi.Query(
+        None, description='Number of requests to show.'),
+    fields: Optional[List[str]] = fastapi.Query(
+        None, description='Fields to get. If None, get all fields.'),
 ) -> List[payloads.RequestPayload]:
     """Gets the list of requests."""
     if request_ids is None:
@@ -1634,9 +1642,15 @@ async def api_status(
                 requests_lib.RequestStatus.PENDING,
                 requests_lib.RequestStatus.RUNNING,
             ]
-        request_tasks = await requests_lib.get_request_tasks_async(
-            req_filter=requests_lib.RequestTaskFilter(status=statuses))
-        return [r.readable_encode() for r in request_tasks]
+        request_tasks = await requests_lib.get_request_tasks_with_fields_async(
+            req_filter=requests_lib.RequestTaskFilter(
+                status=statuses,
+                limit=limit,
+                fields=fields,
+            ),
+            fields=fields,
+        )
+        return requests_lib.encode_requests(request_tasks)
     else:
         encoded_request_tasks = []
         for request_id in request_ids:

sky/server/stream_utils.py

@@ -11,6 +11,7 @@ import fastapi
 from sky import global_user_state
 from sky import sky_logging
 from sky.server.requests import requests as requests_lib
+from sky.utils import common_utils
 from sky.utils import message_utils
 from sky.utils import rich_utils
 from sky.utils import status_lib
@@ -24,7 +25,9 @@ logger = sky_logging.init_logger(__name__)
 _BUFFER_SIZE = 8 * 1024  # 8KB
 _BUFFER_TIMEOUT = 0.02  # 20ms
 _HEARTBEAT_INTERVAL = 30
-_CLUSTER_STATUS_INTERVAL = 1
+
+LONG_REQUEST_POLL_INTERVAL = 1
+DEFAULT_POLL_INTERVAL = 0.1
 
 
 async def _yield_log_file_with_payloads_skipped(
@@ -41,12 +44,14 @@ async def _yield_log_file_with_payloads_skipped(
 
 
 async def log_streamer(
-        request_id: Optional[str],
-        log_path: pathlib.Path,
-        plain_logs: bool = False,
-        tail: Optional[int] = None,
-        follow: bool = True,
-        cluster_name: Optional[str] = None) -> AsyncGenerator[str, None]:
+    request_id: Optional[str],
+    log_path: pathlib.Path,
+    plain_logs: bool = False,
+    tail: Optional[int] = None,
+    follow: bool = True,
+    cluster_name: Optional[str] = None,
+    polling_interval: float = DEFAULT_POLL_INTERVAL
+) -> AsyncGenerator[str, None]:
     """Streams the logs of a request.
 
     Args:
@@ -84,6 +89,11 @@ async def log_streamer(
                        f'scheduled: {request_id}')
         req_status = request_task.status
         req_msg = request_task.status_msg
+        # Slowly back off the database polling up to every 1 second, to avoid
+        # overloading the CPU and DB.
+        backoff = common_utils.Backoff(initial_backoff=polling_interval,
+                                       max_backoff_factor=10,
+                                       multiplier=1.2)
         while req_status < requests_lib.RequestStatus.RUNNING:
             if req_msg is not None:
                 waiting_msg = request_task.status_msg
@@ -99,7 +109,7 @@ async def log_streamer(
             # TODO(aylei): we should use a better mechanism to avoid busy
             # polling the DB, which can be a bottleneck for high-concurrency
             # requests.
-            await asyncio.sleep(0.1)
+            await asyncio.sleep(backoff.current_backoff())
             status_with_msg = await requests_lib.get_request_status_async(
                 request_id, include_msg=True)
             req_status = status_with_msg.status
@@ -111,17 +121,20 @@ async def log_streamer(
 
     async with aiofiles.open(log_path, 'rb') as f:
         async for chunk in _tail_log_file(f, request_id, plain_logs, tail,
-                                          follow, cluster_name):
+                                          follow, cluster_name,
+                                          polling_interval):
             yield chunk
 
 
 async def _tail_log_file(
-        f: aiofiles.threadpool.binary.AsyncBufferedReader,
-        request_id: Optional[str] = None,
-        plain_logs: bool = False,
-        tail: Optional[int] = None,
-        follow: bool = True,
-        cluster_name: Optional[str] = None) -> AsyncGenerator[str, None]:
+    f: aiofiles.threadpool.binary.AsyncBufferedReader,
+    request_id: Optional[str] = None,
+    plain_logs: bool = False,
+    tail: Optional[int] = None,
+    follow: bool = True,
+    cluster_name: Optional[str] = None,
+    polling_interval: float = DEFAULT_POLL_INTERVAL
+) -> AsyncGenerator[str, None]:
     """Tail the opened log file, buffer the lines and flush in chunks."""
 
     if tail is not None:
@@ -137,7 +150,7 @@ async def _tail_log_file(
             yield line_str
 
     last_heartbeat_time = asyncio.get_event_loop().time()
-    last_cluster_status_check_time = asyncio.get_event_loop().time()
+    last_status_check_time = asyncio.get_event_loop().time()
 
     # Buffer the lines in memory and flush them in chunks to improve log
     # tailing throughput.
@@ -167,7 +180,17 @@ async def _tail_log_file(
 
         line: Optional[bytes] = await f.readline()
         if not line:
-            if request_id is not None:
+            # Avoid checking the status too frequently to avoid overloading the
+            # DB.
+            should_check_status = (current_time -
+                                   last_status_check_time) >= polling_interval
+            if not follow:
+                # We will only hit this path once, but we should make sure to
+                # check the status so that we display the final request status
+                # if the request is complete.
+                should_check_status = True
+            if request_id is not None and should_check_status:
+                last_status_check_time = current_time
                 req_status = await requests_lib.get_request_status_async(
                     request_id)
                 if req_status.status > requests_lib.RequestStatus.RUNNING:
@@ -185,20 +208,19 @@ async def _tail_log_file(
                                 ' cancelled\n')
                     break
             if not follow:
+                # The below checks (cluster status, heartbeat) are not needed
+                # for non-follow logs.
                 break
             # Provision logs pass in cluster_name, check cluster status
-            # periodically to see if provisioning is done. We only
-            # check once a second to avoid overloading the DB.
-            check_status = (current_time - last_cluster_status_check_time
-                           ) >= _CLUSTER_STATUS_INTERVAL
-            if cluster_name is not None and check_status:
+            # periodically to see if provisioning is done.
+            if cluster_name is not None and should_check_status:
+                last_status_check_time = current_time
                 cluster_record = await (
                     global_user_state.get_status_from_cluster_name_async(
                         cluster_name))
                 if (cluster_record is None or
                         cluster_record != status_lib.ClusterStatus.INIT):
                     break
-                last_cluster_status_check_time = current_time
             if current_time - last_heartbeat_time >= _HEARTBEAT_INTERVAL:
                 # Currently just used to keep the connection busy, refer to
                 # https://github.com/skypilot-org/skypilot/issues/5750 for
@@ -234,9 +256,22 @@ async def _tail_log_file(
         yield chunk
 
 
+def stream_response_for_long_request(
+    request_id: str,
+    logs_path: pathlib.Path,
+    background_tasks: fastapi.BackgroundTasks,
+) -> fastapi.responses.StreamingResponse:
+    return stream_response(request_id,
+                           logs_path,
+                           background_tasks,
+                           polling_interval=LONG_REQUEST_POLL_INTERVAL)
+
+
 def stream_response(
-    request_id: str, logs_path: pathlib.Path,
-    background_tasks: fastapi.BackgroundTasks
+    request_id: str,
+    logs_path: pathlib.Path,
+    background_tasks: fastapi.BackgroundTasks,
+    polling_interval: float = DEFAULT_POLL_INTERVAL
 ) -> fastapi.responses.StreamingResponse:
 
     async def on_disconnect():
@@ -249,7 +284,7 @@ def stream_response(
     background_tasks.add_task(on_disconnect)
 
     return fastapi.responses.StreamingResponse(
-        log_streamer(request_id, logs_path),
+        log_streamer(request_id, logs_path, polling_interval=polling_interval),
         media_type='text/plain',
         headers={
             'Cache-Control': 'no-cache, no-transform',

sky/utils/common_utils.py

@@ -265,13 +265,16 @@ def get_global_job_id(job_timestamp: str,
 
 class Backoff:
     """Exponential backoff with jittering."""
-    MULTIPLIER = 1.6
     JITTER = 0.4
 
-    def __init__(self, initial_backoff: float = 5, max_backoff_factor: int = 5):
+    def __init__(self,
+                 initial_backoff: float = 5,
+                 max_backoff_factor: int = 5,
+                 multiplier: float = 1.6):
         self._initial = True
         self._backoff = 0.0
         self._initial_backoff = initial_backoff
+        self._multiplier = multiplier
         self._max_backoff = max_backoff_factor * self._initial_backoff
 
     # https://github.com/grpc/grpc/blob/2d4f3c56001cd1e1f85734b2f7c5ce5f2797c38a/doc/connection-backoff.md
@@ -283,7 +286,7 @@ class Backoff:
             self._initial = False
             self._backoff = min(self._initial_backoff, self._max_backoff)
         else:
-            self._backoff = min(self._backoff * self.MULTIPLIER,
+            self._backoff = min(self._backoff * self._multiplier,
                                 self._max_backoff)
         self._backoff += random.uniform(-self.JITTER * self._backoff,
                                         self.JITTER * self._backoff)

{skypilot_nightly-1.0.0.dev20251009.dist-info → skypilot_nightly-1.0.0.dev20251012.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skypilot-nightly
-Version: 1.0.0.dev20251009
+Version: 1.0.0.dev20251012
 Summary: SkyPilot: Run AI on Any Infra — Unified, Faster, Cheaper.
 Author: SkyPilot Team
 License: Apache 2.0
@@ -155,51 +155,51 @@ Requires-Dist: protobuf<7.0.0,>=5.26.1; extra == "server"
 Requires-Dist: aiosqlite; extra == "server"
 Requires-Dist: greenlet; extra == "server"
 Provides-Extra: all
+Requires-Dist: anyio; extra == "all"
 Requires-Dist: nebius>=0.2.47; extra == "all"
-Requires-Dist: websockets; extra == "all"
+Requires-Dist: ecsapi>=0.2.0; extra == "all"
+Requires-Dist: ibm-cos-sdk; extra == "all"
+Requires-Dist: python-dateutil; extra == "all"
 Requires-Dist: azure-core>=1.31.0; extra == "all"
-Requires-Dist: msgraph-sdk; extra == "all"
-Requires-Dist: ibm-cloud-sdk-core; extra == "all"
-Requires-Dist: google-cloud-storage; extra == "all"
+Requires-Dist: aiosqlite; extra == "all"
+Requires-Dist: protobuf<7.0.0,>=5.26.1; extra == "all"
+Requires-Dist: cudo-compute>=0.1.10; extra == "all"
+Requires-Dist: pydo>=0.3.0; extra == "all"
 Requires-Dist: casbin; extra == "all"
-Requires-Dist: ibm-platform-services>=0.48.0; extra == "all"
-Requires-Dist: ecsapi>=0.2.0; extra == "all"
-Requires-Dist: awscli>=1.27.10; extra == "all"
+Requires-Dist: azure-storage-blob>=12.23.1; extra == "all"
+Requires-Dist: boto3>=1.26.1; extra == "all"
 Requires-Dist: sqlalchemy_adapter; extra == "all"
-Requires-Dist: msrestazure; extra == "all"
-Requires-Dist: protobuf<7.0.0,>=5.26.1; extra == "all"
-Requires-Dist: ibm-vpc; extra == "all"
-Requires-Dist: ray[default]>=2.6.1; extra == "all"
-Requires-Dist: anyio; extra == "all"
-Requires-Dist: runpod>=1.6.1; extra == "all"
-Requires-Dist: google-api-python-client>=2.69.0; extra == "all"
+Requires-Dist: passlib; extra == "all"
 Requires-Dist: greenlet; extra == "all"
-Requires-Dist: ibm-cos-sdk; extra == "all"
-Requires-Dist: docker; extra == "all"
+Requires-Dist: msrestazure; extra == "all"
+Requires-Dist: colorama<0.4.5; extra == "all"
 Requires-Dist: azure-common; extra == "all"
-Requires-Dist: python-dateutil; extra == "all"
+Requires-Dist: pyopenssl<24.3.0,>=23.2.0; extra == "all"
+Requires-Dist: websockets; extra == "all"
 Requires-Dist: tomli; python_version < "3.11" and extra == "all"
-Requires-Dist: pyjwt; extra == "all"
-Requires-Dist: botocore>=1.29.10; extra == "all"
-Requires-Dist: azure-core>=1.24.0; extra == "all"
-Requires-Dist: colorama<0.4.5; extra == "all"
-Requires-Dist: passlib; extra == "all"
+Requires-Dist: ray[default]>=2.6.1; extra == "all"
 Requires-Dist: azure-mgmt-compute>=33.0.0; extra == "all"
-Requires-Dist: pyopenssl<24.3.0,>=23.2.0; extra == "all"
+Requires-Dist: google-cloud-storage; extra == "all"
+Requires-Dist: docker; extra == "all"
+Requires-Dist: grpcio>=1.63.0; extra == "all"
+Requires-Dist: msgraph-sdk; extra == "all"
+Requires-Dist: ibm-vpc; extra == "all"
 Requires-Dist: vastai-sdk>=0.1.12; extra == "all"
-Requires-Dist: aiosqlite; extra == "all"
-Requires-Dist: aiohttp; extra == "all"
-Requires-Dist: cudo-compute>=0.1.10; extra == "all"
+Requires-Dist: pyjwt; extra == "all"
 Requires-Dist: pyvmomi==8.0.1.0.2; extra == "all"
-Requires-Dist: boto3>=1.26.1; extra == "all"
-Requires-Dist: pydo>=0.3.0; extra == "all"
+Requires-Dist: botocore>=1.29.10; extra == "all"
 Requires-Dist: azure-cli>=2.65.0; extra == "all"
+Requires-Dist: ibm-platform-services>=0.48.0; extra == "all"
+Requires-Dist: oci; extra == "all"
+Requires-Dist: awscli>=1.27.10; extra == "all"
+Requires-Dist: azure-mgmt-network>=27.0.0; extra == "all"
+Requires-Dist: azure-core>=1.24.0; extra == "all"
 Requires-Dist: azure-identity>=1.19.0; extra == "all"
+Requires-Dist: runpod>=1.6.1; extra == "all"
+Requires-Dist: google-api-python-client>=2.69.0; extra == "all"
+Requires-Dist: ibm-cloud-sdk-core; extra == "all"
 Requires-Dist: kubernetes!=32.0.0,>=20.0.0; extra == "all"
-Requires-Dist: azure-storage-blob>=12.23.1; extra == "all"
-Requires-Dist: azure-mgmt-network>=27.0.0; extra == "all"
-Requires-Dist: grpcio>=1.63.0; extra == "all"
-Requires-Dist: oci; extra == "all"
+Requires-Dist: aiohttp; extra == "all"
 Dynamic: author
 Dynamic: classifier
 Dynamic: description
@@ -249,10 +249,11 @@ Dynamic: summary
 ----
 
 :fire: *News* :fire:
+- [Oct 2025] Run large-scale **LLM training with TorchTitan** on any AI infra: [**example**](./llm/torchtitan/)
+- [Sep 2025] Scaling AI infrastructure at Abridge - **10x faster development** with SkyPilot: [**blog**](https://blog.skypilot.co/abridge/)
+- [Sep 2025] Network and Storage Benchmarks for LLM training on the cloud: [**blog**](https://maknee.github.io/blog/2025/Network-And-Storage-Training-Skypilot/)
 - [Aug 2025] Serve and finetune **OpenAI GPT-OSS models** (gpt-oss-120b, gpt-oss-20b) with one command on any infra: [**serve**](./llm/gpt-oss/) + [**LoRA and full finetuning**](./llm/gpt-oss-finetuning/)
-- [Jul 2025] Run large-scale **LLM training with TorchTitan** on any cloud: [**example**](./llm/torchtitan/)
 - [Jul 2025] Run distributed **RL training for LLMs** with Verl (PPO, GRPO) on any cloud: [**example**](./llm/verl/)
-- [Jul 2025] 🎉 SkyPilot v0.10.0 released! [**blog post**](https://blog.skypilot.co/announcing-skypilot-0.10.0/), [**release notes**](https://github.com/skypilot-org/skypilot/releases/tag/v0.10.0)
 - [Jul 2025] Finetune **Llama4** on any distributed cluster/cloud: [**example**](./llm/llama-4-finetuning/)
 - [Jul 2025] Two-part blog series, `The Evolution of AI Job Orchestration`: (1) [Running AI jobs on GPU Neoclouds](https://blog.skypilot.co/ai-job-orchestration-pt1-gpu-neoclouds/), (2) [The AI-Native Control Plane & Orchestration that Finally Works for ML](https://blog.skypilot.co/ai-job-orchestration-pt2-ai-control-plane/)
 - [Apr 2025] Spin up **Qwen3** on your cluster/cloud: [**example**](./llm/qwen/)