skypilot-nightly 1.0.0.dev20250708__py3-none-any.whl → 1.0.0.dev20250710__py3-none-any.whl

sky/jobs/state.py

@@ -77,6 +77,7 @@ spot_table = sqlalchemy.Table(
     sqlalchemy.Column('task_name', sqlalchemy.Text),
     sqlalchemy.Column('specs', sqlalchemy.Text),
     sqlalchemy.Column('local_log_file', sqlalchemy.Text, server_default=None),
+    sqlalchemy.Column('metadata', sqlalchemy.Text, server_default='{}'),
 )
 
 job_info_table = sqlalchemy.Table(
@@ -131,7 +132,7 @@ def create_table():
             # is not critical and is likely to be enabled by other processes.
 
     # Create tables if they don't exist
-    Base.metadata.create_all(bind=_SQLALCHEMY_ENGINE)
+    db_utils.add_tables_to_db_sqlalchemy(Base.metadata, _SQLALCHEMY_ENGINE)
 
     # Backward compatibility: add columns that not exist in older databases
     with orm.Session(_SQLALCHEMY_ENGINE) as session:
@@ -170,6 +171,14 @@ def create_table():
             sqlalchemy.Text(),
             default_statement='DEFAULT NULL')
 
+        db_utils.add_column_to_table_sqlalchemy(
+            session,
+            'spot',
+            'metadata',
+            sqlalchemy.Text(),
+            default_statement='DEFAULT \'{}\'',
+            value_to_replace_existing_entries='{}')
+
         db_utils.add_column_to_table_sqlalchemy(session, 'job_info',
                                                 'schedule_state',
                                                 sqlalchemy.Text())
@@ -219,7 +228,8 @@ def initialize_and_get_db() -> sqlalchemy.engine.Engine:
                 conn_string = skypilot_config.get_nested(('db',), None)
             if conn_string:
                 logger.debug(f'using db URI from {conn_string}')
-                _SQLALCHEMY_ENGINE = sqlalchemy.create_engine(conn_string)
+                _SQLALCHEMY_ENGINE = sqlalchemy.create_engine(
+                    conn_string, poolclass=sqlalchemy.NullPool)
             else:
                 db_path = os.path.expanduser('~/.sky/spot_jobs.db')
                 pathlib.Path(db_path).parents[0].mkdir(parents=True,
@@ -272,6 +282,7 @@ def _get_jobs_dict(r: 'row.RowMapping') -> Dict[str, Any]:
         'task_name': r['task_name'],
         'specs': r['specs'],
         'local_log_file': r['local_log_file'],
+        'metadata': r['metadata'],
         # columns from job_info table (some may be None for legacy jobs)
         '_job_info_job_id': r[job_info_table.c.spot_job_id
                              ],  # ambiguous, use table.column
@@ -544,20 +555,28 @@ def set_job_info_without_job_id(name: str, workspace: str,
         if (_SQLALCHEMY_ENGINE.dialect.name ==
                 db_utils.SQLAlchemyDialect.SQLITE.value):
             result = session.execute(insert_stmt)
+            ret = result.lastrowid
             session.commit()
-            return result.lastrowid
+            return ret
         elif (_SQLALCHEMY_ENGINE.dialect.name ==
               db_utils.SQLAlchemyDialect.POSTGRESQL.value):
             result = session.execute(
                 insert_stmt.returning(job_info_table.c.spot_job_id))
+            ret = result.scalar()
             session.commit()
-            return result.scalar()
+            return ret
         else:
             raise ValueError('Unsupported database dialect')
 
 
 @_init_db
-def set_pending(job_id: int, task_id: int, task_name: str, resources_str: str):
+def set_pending(
+    job_id: int,
+    task_id: int,
+    task_name: str,
+    resources_str: str,
+    metadata: str,
+):
     """Set the task to pending state."""
     assert _SQLALCHEMY_ENGINE is not None
     with orm.Session(_SQLALCHEMY_ENGINE) as session:
@@ -567,6 +586,7 @@ def set_pending(job_id: int, task_id: int, task_name: str, resources_str: str):
                 task_id=task_id,
                 task_name=task_name,
                 resources=resources_str,
+                metadata=metadata,
                 status=ManagedJobStatus.PENDING.value,
             ))
         session.commit()
@@ -1192,38 +1212,40 @@ def get_managed_jobs(job_id: Optional[int] = None) -> List[Dict[str, Any]]:
     # Note: we will get the user_hash here, but don't try to call
     # global_user_state.get_user() on it. This runs on the controller, which may
     # not have the user info. Prefer to do it on the API server side.
+    query = sqlalchemy.select(spot_table, job_info_table).select_from(
+        spot_table.outerjoin(
+            job_info_table,
+            spot_table.c.spot_job_id == job_info_table.c.spot_job_id))
+    if job_id is not None:
+        query = query.where(spot_table.c.spot_job_id == job_id)
+    query = query.order_by(spot_table.c.spot_job_id.desc(),
+                           spot_table.c.task_id.asc())
+    rows = None
     with orm.Session(_SQLALCHEMY_ENGINE) as session:
-        query = sqlalchemy.select(spot_table, job_info_table).select_from(
-            spot_table.outerjoin(
-                job_info_table,
-                spot_table.c.spot_job_id == job_info_table.c.spot_job_id))
-        if job_id is not None:
-            query = query.where(spot_table.c.spot_job_id == job_id)
-        query = query.order_by(spot_table.c.spot_job_id.desc(),
-                               spot_table.c.task_id.asc())
         rows = session.execute(query).fetchall()
-        jobs = []
-        for row in rows:
-            job_dict = _get_jobs_dict(row._mapping)  # pylint: disable=protected-access
-            job_dict['status'] = ManagedJobStatus(job_dict['status'])
-            job_dict['schedule_state'] = ManagedJobScheduleState(
-                job_dict['schedule_state'])
-            if job_dict['job_name'] is None:
-                job_dict['job_name'] = job_dict['task_name']
-
-            # Add user YAML content for managed jobs.
-            yaml_path = job_dict.get('original_user_yaml_path')
-            if yaml_path:
-                try:
-                    with open(yaml_path, 'r', encoding='utf-8') as f:
-                        job_dict['user_yaml'] = f.read()
-                except (FileNotFoundError, IOError, OSError):
-                    job_dict['user_yaml'] = None
-            else:
+    jobs = []
+    for row in rows:
+        job_dict = _get_jobs_dict(row._mapping)  # pylint: disable=protected-access
+        job_dict['status'] = ManagedJobStatus(job_dict['status'])
+        job_dict['schedule_state'] = ManagedJobScheduleState(
+            job_dict['schedule_state'])
+        if job_dict['job_name'] is None:
+            job_dict['job_name'] = job_dict['task_name']
+        job_dict['metadata'] = json.loads(job_dict['metadata'])
+
+        # Add user YAML content for managed jobs.
+        yaml_path = job_dict.get('original_user_yaml_path')
+        if yaml_path:
+            try:
+                with open(yaml_path, 'r', encoding='utf-8') as f:
+                    job_dict['user_yaml'] = f.read()
+            except (FileNotFoundError, IOError, OSError):
                 job_dict['user_yaml'] = None
+        else:
+            job_dict['user_yaml'] = None
 
-            jobs.append(job_dict)
-        return jobs
+        jobs.append(job_dict)
+    return jobs
 
 
 @_init_db

sky/jobs/utils.py

@@ -1249,7 +1249,14 @@ def format_job_table(
     ]
     if show_all:
         # TODO: move SCHED. STATE to a separate flag (e.g. --debug)
-        columns += ['STARTED', 'INFRA', 'RESOURCES', 'SCHED. STATE', 'DETAILS']
+        columns += [
+            'STARTED',
+            'INFRA',
+            'RESOURCES',
+            'SCHED. STATE',
+            'DETAILS',
+            'GIT_COMMIT',
+        ]
     if tasks_have_k8s_user:
         columns.insert(0, 'USER')
     job_table = log_utils.create_table(columns)
@@ -1362,6 +1369,7 @@ def format_job_table(
                     '-',
                     job_tasks[0]['schedule_state'],
                     generate_details(details, failure_reason),
+                    job_tasks[0].get('metadata', {}).get('git_commit', '-'),
                 ])
             if tasks_have_k8s_user:
                 job_values.insert(0, job_tasks[0].get('user', '-'))
@@ -1427,6 +1435,8 @@ def format_job_table(
                     generate_details(task.get('details'),
                                      task['failure_reason']),
                 ])
+
+                values.append(task.get('metadata', {}).get('git_commit', '-'))
             if tasks_have_k8s_user:
                 values.insert(0, task.get('user', '-'))
             job_table.add_row(values)
@@ -1511,6 +1521,22 @@ class ManagedJobCodeGen:
         """)
         return cls._build(code)
 
+    @classmethod
+    def get_version_and_job_table(cls) -> str:
+        """Generate code to get controller version and raw job table."""
+        code = textwrap.dedent("""\
+        from sky.skylet import constants as controller_constants
+
+        # Get controller version
+        controller_version = controller_constants.SKYLET_VERSION
+        print(f"controller_version:{controller_version}", flush=True)
+
+        # Get and print raw job table (load_managed_job_queue can parse this directly)
+        job_table = utils.dump_managed_job_queue()
+        print(job_table, flush=True)
+        """)
+        return cls._build(code)
+
     @classmethod
     def get_all_job_ids_by_name(cls, job_name: Optional[str]) -> str:
         code = textwrap.dedent(f"""\
@@ -1565,8 +1591,13 @@ class ManagedJobCodeGen:
             resources_str = backend_utils.get_task_resources_str(
                 task, is_managed_job=True)
             code += textwrap.dedent(f"""\
-                managed_job_state.set_pending({job_id}, {task_id},
-                                  {task.name!r}, {resources_str!r})
+                if managed_job_version < 7:
+                    managed_job_state.set_pending({job_id}, {task_id},
+                                    {task.name!r}, {resources_str!r})
+                else:
+                    managed_job_state.set_pending({job_id}, {task_id},
+                                    {task.name!r}, {resources_str!r},
+                                    {task.metadata_json!r})
                 """)
         return cls._build(code)
 

sky/provision/kubernetes/instance.py

@@ -825,6 +825,23 @@ def _create_pods(region: str, cluster_name_on_cloud: str,
                 return
             pod_spec_copy['metadata']['name'] = pod_name
             pod_spec_copy['metadata']['labels']['component'] = pod_name
+
+        # We need to keep the following fields in the pod spec to be same for
+        # head and worker pods.
+        # So that Kueue can merge them into a single PodSet when creating
+        # ProvisioningRequest to trigger scale up of the cluster autoscaler,
+        # this is especially required for DWS queued provisioning mode in GKE.
+        #  spec.containers[*].resources.requests
+        #  spec.initContainers[*].resources.requests
+        #  spec.resources
+        #  spec.nodeSelector
+        #  spec.tolerations
+        #  spec.affinity
+        #  resourceClaims
+        # Refer to the following links for more details:
+        # https://cloud.google.com/kubernetes-engine/docs/how-to/provisioningrequest#define_a_provisioningrequest_object # pylint: disable=line-too-long
+        # https://kueue.sigs.k8s.io/docs/admission-check-controllers/provisioning/#podset-merge-policy # pylint: disable=line-too-long
+        if config.count > 1:
             # For multi-node support, we put a soft-constraint to schedule
             # worker pods on different nodes than the head pod.
             # This is not set as a hard constraint because if different nodes

sky/provision/kubernetes/utils.py

@@ -313,6 +313,9 @@ def get_gke_accelerator_name(accelerator: str) -> str:
         # A100-80GB, L4, H100-80GB and H100-MEGA-80GB
         # have a different name pattern.
         return 'nvidia-{}'.format(accelerator.lower())
+    elif accelerator == 'H200':
+        # H200s on GCP use this label format
+        return 'nvidia-h200-141gb'
     elif accelerator.startswith('tpu-'):
         return accelerator
     else:
@@ -482,6 +485,8 @@ class GKELabelFormatter(GPULabelFormatter):
                 # we map H100 ---> H100-80GB and keep H100-MEGA-80GB
                 # to distinguish between a3-high and a3-mega instances
                 return 'H100'
+            elif acc == 'H200-141GB':
+                return 'H200'
             return acc
         elif is_tpu_on_gke(value):
             return value

sky/provision/provisioner.py

@@ -22,11 +22,13 @@ from sky import sky_logging
 from sky import skypilot_config
 from sky.adaptors import aws
 from sky.backends import backend_utils
+from sky.jobs.server import utils as server_jobs_utils
 from sky.provision import common as provision_common
 from sky.provision import instance_setup
 from sky.provision import logging as provision_logging
 from sky.provision import metadata_utils
 from sky.skylet import constants
+from sky.utils import common
 from sky.utils import common_utils
 from sky.utils import message_utils
 from sky.utils import resources_utils
@@ -502,6 +504,24 @@ def _post_provision_setup(
             logger.info(f'{ux_utils.INDENT_LAST_SYMBOL}{colorama.Style.DIM}'
                         f'Docker container is up.{colorama.Style.RESET_ALL}')
 
+        # Check version compatibility for jobs controller clusters
+        if cluster_name.display_name.startswith(common.JOB_CONTROLLER_PREFIX):
+            # TODO(zeping): remove this in v0.12.0
+            # This only happens in upgrade from <0.9.3 to > 0.10.0
+            # After 0.10.0 no incompatibility issue
+            # See https://github.com/skypilot-org/skypilot/pull/6096
+            # For more details
+            status.update(
+                ux_utils.spinner_message(
+                    'Checking controller version compatibility'))
+            try:
+                server_jobs_utils.check_version_mismatch_and_non_terminal_jobs()
+            except exceptions.ClusterNotUpError:
+                # Controller is not up yet during initial provisioning, that
+                # also means no non-terminal jobs, so no incompatibility in
+                # this case.
+                pass
+
         # We mount the metadata with sky wheel for speedup.
         # NOTE: currently we mount all credentials for all nodes, because
         # (1) jobs controllers need permission to launch/down nodes of

sky/server/metrics.py

@@ -1,6 +1,5 @@
 """Instrumentation for the API server."""
 
-import asyncio
 import os
 import time
 
@@ -50,7 +49,7 @@ async def metrics() -> fastapi.Response:
                             headers={'Cache-Control': 'no-cache'})
 
 
-def run_metrics_server(host: str, port: int):
+def build_metrics_server(host: str, port: int) -> uvicorn.Server:
     metrics_config = uvicorn.Config(
         'sky.server.metrics:metrics_app',
         host=host,
@@ -58,7 +57,7 @@ def run_metrics_server(host: str, port: int):
         workers=1,
     )
     metrics_server_instance = uvicorn.Server(metrics_config)
-    asyncio.run(metrics_server_instance.serve())
+    return metrics_server_instance
 
 
 def _get_status_code_group(status_code: int) -> str:

sky/server/requests/executor.py

@@ -399,11 +399,8 @@ def _request_execution_wrapper(request_id: str,
                         f'{common_utils.format_exception(e)}')
             return
         else:
-            with api_requests.update_request(request_id) as request_task:
-                assert request_task is not None, request_id
-                request_task.status = api_requests.RequestStatus.SUCCEEDED
-                if not ignore_return_value:
-                    request_task.set_return_value(return_value)
+            api_requests.set_request_succeeded(
+                request_id, return_value if not ignore_return_value else None)
             _restore_output(original_stdout, original_stderr)
             logger.info(f'Request {request_id} finished')
 

sky/server/requests/payloads.py

@@ -710,3 +710,4 @@ class RequestPayload(BasePayload):
     cluster_name: Optional[str] = None
     status_msg: Optional[str] = None
     should_retry: bool = False
+    finished_at: Optional[float] = None

sky/server/requests/requests.py

@@ -1,4 +1,5 @@
 """Utilities for REST API."""
+import asyncio
 import contextlib
 import dataclasses
 import enum
@@ -20,6 +21,7 @@ import filelock
 from sky import exceptions
 from sky import global_user_state
 from sky import sky_logging
+from sky import skypilot_config
 from sky.server import common as server_common
 from sky.server import constants as server_constants
 from sky.server.requests import payloads
@@ -29,6 +31,7 @@ from sky.utils import common
 from sky.utils import common_utils
 from sky.utils import db_utils
 from sky.utils import env_options
+from sky.utils import subprocess_utils
 from sky.utils import ux_utils
 
 logger = sky_logging.init_logger(__name__)
@@ -39,8 +42,11 @@ COL_CLUSTER_NAME = 'cluster_name'
 COL_USER_ID = 'user_id'
 COL_STATUS_MSG = 'status_msg'
 COL_SHOULD_RETRY = 'should_retry'
+COL_FINISHED_AT = 'finished_at'
 REQUEST_LOG_PATH_PREFIX = '~/sky_logs/api_server/requests'
 
+DEFAULT_REQUESTS_RETENTION_HOURS = 24  # 1 day
+
 # TODO(zhwu): For scalability, there are several TODOs:
 # [x] Have a way to queue requests.
 # [ ] Move logs to persistent place.
@@ -64,6 +70,10 @@ class RequestStatus(enum.Enum):
         color = _STATUS_TO_COLOR[self]
         return f'{color}{self.value}{colorama.Style.RESET_ALL}'
 
+    @classmethod
+    def finished_status(cls) -> List['RequestStatus']:
+        return [cls.SUCCEEDED, cls.FAILED, cls.CANCELLED]
+
 
 _STATUS_TO_COLOR = {
     RequestStatus.PENDING: colorama.Fore.BLUE,
@@ -88,6 +98,7 @@ REQUEST_COLUMNS = [
     COL_USER_ID,
     COL_STATUS_MSG,
     COL_SHOULD_RETRY,
+    COL_FINISHED_AT,
 ]
 
 
@@ -120,6 +131,8 @@ class Request:
     status_msg: Optional[str] = None
     # Whether the request should be retried.
     should_retry: bool = False
+    # When the request finished.
+    finished_at: Optional[float] = None
 
     @property
     def log_path(self) -> pathlib.Path:
@@ -206,6 +219,7 @@ class Request:
             cluster_name=self.cluster_name,
             status_msg=self.status_msg,
             should_retry=self.should_retry,
+            finished_at=self.finished_at,
         )
 
     def encode(self) -> payloads.RequestPayload:
@@ -228,6 +242,7 @@ class Request:
                 cluster_name=self.cluster_name,
                 status_msg=self.status_msg,
                 should_retry=self.should_retry,
+                finished_at=self.finished_at,
             )
         except (TypeError, ValueError) as e:
             # The error is unexpected, so we don't suppress the stack trace.
@@ -260,6 +275,7 @@ class Request:
                 cluster_name=payload.cluster_name,
                 status_msg=payload.status_msg,
                 should_retry=payload.should_retry,
+                finished_at=payload.finished_at,
             )
         except (TypeError, ValueError) as e:
             logger.error(
@@ -439,6 +455,7 @@ def kill_requests(request_ids: Optional[List[str]] = None,
                 #   process for each request.
                 os.kill(request_record.pid, signal.SIGTERM)
             request_record.status = RequestStatus.CANCELLED
+            request_record.finished_at = time.time()
             cancelled_request_ids.append(request_id)
     return cancelled_request_ids
 
@@ -474,13 +491,16 @@ def create_table(cursor, conn):
         schedule_type TEXT,
         {COL_USER_ID} TEXT,
         {COL_STATUS_MSG} TEXT,
-        {COL_SHOULD_RETRY} INTEGER
+        {COL_SHOULD_RETRY} INTEGER,
+        {COL_FINISHED_AT} REAL
         )""")
 
     db_utils.add_column_to_table(cursor, conn, REQUEST_TABLE, COL_STATUS_MSG,
                                  'TEXT')
     db_utils.add_column_to_table(cursor, conn, REQUEST_TABLE, COL_SHOULD_RETRY,
                                  'INTEGER')
+    db_utils.add_column_to_table(cursor, conn, REQUEST_TABLE, COL_FINISHED_AT,
+                                 'REAL')
 
 
 _DB = None
@@ -583,6 +603,7 @@ def get_request_tasks(
     user_id: Optional[str] = None,
     exclude_request_names: Optional[List[str]] = None,
     include_request_names: Optional[List[str]] = None,
+    finished_before: Optional[float] = None,
 ) -> List[Request]:
     """Get a list of requests that match the given filters.
 
@@ -595,6 +616,8 @@ def get_request_tasks(
             If None, all users are included.
         include_request_names: a list of request names to filter on.
             Mutually exclusive with exclude_request_names.
+        finished_before: if provided, only include requests finished before this
+            timestamp.
 
     Raises:
         ValueError: If both exclude_request_names and include_request_names are
@@ -606,7 +629,7 @@ def get_request_tasks(
             'provided, not both.')
 
     filters = []
-    filter_params = []
+    filter_params: List[Any] = []
     if status is not None:
         status_list_str = ','.join(repr(status.value) for status in status)
         filters.append(f'status IN ({status_list_str})')
@@ -624,6 +647,9 @@ def get_request_tasks(
         request_names_str = ','.join(
             repr(name) for name in include_request_names)
         filters.append(f'name IN ({request_names_str})')
+    if finished_before is not None:
+        filters.append('finished_at < ?')
+        filter_params.append(finished_before)
     assert _DB is not None
     with _DB.conn:
         cursor = _DB.conn.cursor()
@@ -665,19 +691,83 @@ def set_request_failed(request_id: str, e: BaseException) -> None:
     with update_request(request_id) as request_task:
         assert request_task is not None, request_id
         request_task.status = RequestStatus.FAILED
+        request_task.finished_at = time.time()
         request_task.set_error(e)
 
 
-def set_request_succeeded(request_id: str, result: Any) -> None:
+def set_request_succeeded(request_id: str, result: Optional[Any]) -> None:
     """Set a request to succeeded and populate the result."""
     with update_request(request_id) as request_task:
         assert request_task is not None, request_id
         request_task.status = RequestStatus.SUCCEEDED
-        request_task.set_return_value(result)
+        request_task.finished_at = time.time()
+        if result is not None:
+            request_task.set_return_value(result)
 
 
 def set_request_cancelled(request_id: str) -> None:
     """Set a request to cancelled."""
     with update_request(request_id) as request_task:
         assert request_task is not None, request_id
+        request_task.finished_at = time.time()
         request_task.status = RequestStatus.CANCELLED
+
+
+@init_db
+def _delete_requests(requests: List[Request]):
+    """Clean up requests by their IDs."""
+    id_list_str = ','.join(repr(req.request_id) for req in requests)
+    assert _DB is not None
+    with _DB.conn:
+        cursor = _DB.conn.cursor()
+        cursor.execute(
+            f'DELETE FROM {REQUEST_TABLE} WHERE request_id IN ({id_list_str})')
+
+
+def clean_finished_requests_with_retention(retention_seconds: int):
+    """Clean up finished requests older than the retention period.
+
+    This function removes old finished requests (SUCCEEDED, FAILED, CANCELLED)
+    from the database and cleans up their associated log files.
+
+    Args:
+        retention_seconds: Requests older than this many seconds will be
+            deleted.
+    """
+    reqs = get_request_tasks(status=RequestStatus.finished_status(),
+                             finished_before=time.time() - retention_seconds)
+
+    subprocess_utils.run_in_parallel(
+        func=lambda req: req.log_path.unlink(missing_ok=True),
+        args=reqs,
+        num_threads=len(reqs))
+
+    _delete_requests(reqs)
+
+    # To avoid leakage of the log file, logs must be deleted before the
+    # request task in the database.
+    logger.info(f'Cleaned up {len(reqs)} finished requests '
+                f'older than {retention_seconds} seconds')
+
+
+async def requests_gc_daemon():
+    """Garbage collect finished requests periodically."""
+    while True:
+        logger.info('Running requests GC daemon...')
+        # Use the latest config.
+        skypilot_config.reload_config()
+        retention_seconds = skypilot_config.get_nested(
+            ('api_server', 'requests_retention_hours'),
+            DEFAULT_REQUESTS_RETENTION_HOURS) * 3600
+        try:
+            # Negative value disables the requests GC
+            if retention_seconds >= 0:
+                clean_finished_requests_with_retention(retention_seconds)
+        except asyncio.CancelledError:
+            logger.info('Requests GC daemon cancelled')
+            break
+        except Exception as e:  # pylint: disable=broad-except
+            logger.error(f'Error running requests GC daemon: {e}')
+        # Run the daemon at most once every hour to avoid too frequent
+        # cleanup.
+        await asyncio.sleep(max(retention_seconds, 3600))

sky/server/server.py

@@ -26,6 +26,7 @@ import fastapi
 from fastapi.middleware import cors
 from passlib.hash import apr_md5_crypt
 import starlette.middleware.base
+import uvloop
 
 import sky
 from sky import catalog
@@ -1461,6 +1462,12 @@ async def stream(
             raise fastapi.HTTPException(
                 status_code=404, detail=f'Request {request_id!r} not found')
         log_path_to_stream = request_task.log_path
+        if not log_path_to_stream.exists():
+            # The log file might be deleted by the request GC daemon but the
+            # request task is still in the database.
+            raise fastapi.HTTPException(
+                status_code=404,
+                detail=f'Log of request {request_id!r} has been deleted')
     else:
         assert log_path is not None, (request_id, log_path)
         if log_path == constants.API_SERVER_LOGS:
@@ -1775,13 +1782,18 @@ if __name__ == '__main__':
 
     queue_server: Optional[multiprocessing.Process] = None
     workers: List[executor.RequestWorker] = []
+    # Global background tasks that will be scheduled in a separate event loop.
+    global_tasks: List[asyncio.Task] = []
     try:
+        background = uvloop.new_event_loop()
         if os.environ.get(constants.ENV_VAR_SERVER_METRICS_ENABLED):
-            metrics_thread = threading.Thread(target=metrics.run_metrics_server,
-                                              args=(cmd_args.host,
-                                                    cmd_args.metrics_port),
-                                              daemon=True)
-            metrics_thread.start()
+            metrics_server = metrics.build_metrics_server(
+                cmd_args.host, cmd_args.metrics_port)
+            global_tasks.append(background.create_task(metrics_server.serve()))
+        global_tasks.append(
+            background.create_task(requests_lib.requests_gc_daemon()))
+        threading.Thread(target=background.run_forever, daemon=True).start()
+
         queue_server, workers = executor.start(config)
 
         logger.info(f'Starting SkyPilot API server, workers={num_workers}')
@@ -1799,6 +1811,8 @@ if __name__ == '__main__':
     finally:
         logger.info('Shutting down SkyPilot API server...')
 
+        for gt in global_tasks:
+            gt.cancel()
         subprocess_utils.run_in_parallel(lambda worker: worker.cancel(),
                                          workers,
                                          num_threads=len(workers))

sky/server/uvicorn.py

@@ -150,7 +150,10 @@ class Server(uvicorn.Server):
             if req is None:
                 return
             if req.pid is not None:
-                os.kill(req.pid, signal.SIGTERM)
+                try:
+                    os.kill(req.pid, signal.SIGTERM)
+                except ProcessLookupError:
+                    logger.debug(f'Process {req.pid} already finished.')
             req.status = requests_lib.RequestStatus.CANCELLED
             req.should_retry = True
         logger.info(

sky/skylet/constants.py

@@ -89,18 +89,13 @@ TASK_ID_LIST_ENV_VAR = f'{SKYPILOT_ENV_VAR_PREFIX}TASK_IDS'
 # cluster yaml is updated.
 #
 # TODO(zongheng,zhanghao): make the upgrading of skylet automatic?
-SKYLET_VERSION = '14'
+SKYLET_VERSION = '15'
 # The version of the lib files that skylet/jobs use. Whenever there is an API
 # change for the job_lib or log_lib, we need to bump this version, so that the
 # user can be notified to update their SkyPilot version on the remote cluster.
 SKYLET_LIB_VERSION = 3
 SKYLET_VERSION_FILE = '~/.sky/skylet_version'
 
-# `sky jobs dashboard`-related
-#
-# Port on the remote jobs controller that the dashboard is running on.
-SPOT_DASHBOARD_REMOTE_PORT = 5000
-
 # Docker default options
 DEFAULT_DOCKER_CONTAINER_NAME = 'sky_container'
 DEFAULT_DOCKER_PORT = 10022