skypilot-nightly 1.0.0.dev20250717__py3-none-any.whl → 1.0.0.dev20250720__py3-none-any.whl

sky/schemas/db/script.py.mako

@@ -0,0 +1,28 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    ${downgrades if downgrades else "pass"}

sky/schemas/db/skypilot_config/001_initial_schema.py

@@ -0,0 +1,30 @@
+"""Initial schema for sky config database
+
+Revision ID: 001
+Revises:
+Create Date: 2024-01-01 12:00:00.000000
+
+"""
+# pylint: disable=invalid-name
+from alembic import op
+
+from sky.skypilot_config import Base
+from sky.utils.db import db_utils
+
+# revision identifiers, used by Alembic.
+revision = '001'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    """Create initial schema for config_yaml table"""
+    with op.get_context().autocommit_block():
+        # Create all tables with their current schema
+        db_utils.add_tables_to_db_sqlalchemy(Base.metadata, op.get_bind())
+
+
+def downgrade():
+    """Drop all tables"""
+    Base.metadata.drop_all(bind=op.get_bind())

sky/schemas/db/spot_jobs/001_initial_schema.py

@@ -0,0 +1,97 @@
+"""Initial schema for spot jobs database with backwards compatibility columns
+
+Revision ID: 001
+Revises:
+Create Date: 2024-01-01 12:00:00.000000
+
+"""
+# pylint: disable=invalid-name
+import json
+
+from alembic import op
+import sqlalchemy as sa
+
+from sky.jobs.state import Base
+from sky.skylet import constants
+from sky.utils.db import db_utils
+
+# revision identifiers, used by Alembic.
+revision = '001'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    """Create initial schema and add all backwards compatibility columns"""
+    with op.get_context().autocommit_block():
+        # Create all tables with their current schema
+        db_utils.add_tables_to_db_sqlalchemy(Base.metadata, op.get_bind())
+
+        # Add backwards compatibility columns using helper function that matches
+        # original add_column_to_table_sqlalchemy behavior exactly
+
+        # Spot table columns
+        db_utils.add_column_to_table_alembic('spot', 'failure_reason',
+                                             sa.Text())
+        db_utils.add_column_to_table_alembic('spot',
+                                             'spot_job_id',
+                                             sa.Integer(),
+                                             copy_from='job_id')
+        db_utils.add_column_to_table_alembic(
+            'spot',
+            'task_id',
+            sa.Integer(),
+            server_default='0',
+            value_to_replace_existing_entries=0)
+        db_utils.add_column_to_table_alembic('spot',
+                                             'task_name',
+                                             sa.Text(),
+                                             copy_from='job_name')
+        db_utils.add_column_to_table_alembic(
+            'spot',
+            'specs',
+            sa.Text(),
+            value_to_replace_existing_entries=json.dumps(
+                {'max_restarts_on_errors': 0}))
+        db_utils.add_column_to_table_alembic('spot', 'local_log_file',
+                                             sa.Text())
+        db_utils.add_column_to_table_alembic(
+            'spot',
+            'metadata',
+            sa.Text(),
+            server_default='{}',
+            value_to_replace_existing_entries='{}')
+
+        # Job info table columns
+        db_utils.add_column_to_table_alembic('job_info', 'schedule_state',
+                                             sa.Text())
+        db_utils.add_column_to_table_alembic('job_info', 'controller_pid',
+                                             sa.Integer())
+        db_utils.add_column_to_table_alembic('job_info', 'dag_yaml_path',
+                                             sa.Text())
+        db_utils.add_column_to_table_alembic('job_info', 'env_file_path',
+                                             sa.Text())
+        db_utils.add_column_to_table_alembic('job_info', 'user_hash', sa.Text())
+        db_utils.add_column_to_table_alembic(
+            'job_info',
+            'workspace',
+            sa.Text(),
+            value_to_replace_existing_entries=constants.
+            SKYPILOT_DEFAULT_WORKSPACE)
+        db_utils.add_column_to_table_alembic(
+            'job_info',
+            'priority',
+            sa.Integer(),
+            server_default=str(constants.DEFAULT_PRIORITY),
+            value_to_replace_existing_entries=constants.DEFAULT_PRIORITY)
+        db_utils.add_column_to_table_alembic('job_info', 'entrypoint',
+                                             sa.Text())
+        db_utils.add_column_to_table_alembic('job_info',
+                                             'original_user_yaml_path',
+                                             sa.Text())
+
+
+def downgrade():
+    """Drop all tables"""
+    Base.metadata.drop_all(bind=op.get_bind())

sky/serve/client/sdk.py

@@ -292,12 +292,13 @@ def status(
 
 @usage_lib.entrypoint
 @server_common.check_server_healthy_or_start
-@rest.retry_on_server_unavailable()
+@rest.retry_transient_errors()
 def tail_logs(service_name: str,
               target: Union[str, 'serve_utils.ServiceComponent'],
               replica_id: Optional[int] = None,
               follow: bool = True,
-              output_stream: Optional['io.TextIOBase'] = None) -> None:
+              output_stream: Optional['io.TextIOBase'] = None,
+              tail: Optional[int] = None) -> None:
     """Tails logs for a service.
 
     Usage:
@@ -367,6 +368,7 @@ def tail_logs(service_name: str,
         target=target,
         replica_id=replica_id,
         follow=follow,
+        tail=tail,
     )
     response = server_common.make_authenticated_request(
         'POST',
@@ -390,7 +392,8 @@ def sync_down_logs(service_name: str,
                        str, 'serve_utils.ServiceComponent',
                        List[Union[str,
                                   'serve_utils.ServiceComponent']]]] = None,
-                   replica_ids: Optional[List[int]] = None) -> None:
+                   replica_ids: Optional[List[int]] = None,
+                   tail: Optional[int] = None) -> None:
     """Sync down logs from the service components to a local directory.
 
     This function syncs logs from the specified service components (controller,
@@ -429,6 +432,7 @@ def sync_down_logs(service_name: str,
         local_dir=local_dir,
         targets=targets,
         replica_ids=replica_ids,
+        tail=tail,
     )
     response = server_common.make_authenticated_request(
         'POST',

sky/serve/controller.py

@@ -156,9 +156,13 @@ class SkyServeController:
                 return responses.JSONResponse(content={'message': 'Success'},
                                               status_code=200)
             except Exception as e:  # pylint: disable=broad-except
-                logger.error(f'Error in update_service: '
-                             f'{common_utils.format_exception(e)}')
-                return responses.JSONResponse(content={'message': 'Error'},
+                exception_str = common_utils.format_exception(e)
+                logger.error(f'Error in update_service: {exception_str}')
+                return responses.JSONResponse(content={
+                    'message': 'Error',
+                    'exception': exception_str,
+                    'traceback': traceback.format_exc()
+                },
                                               status_code=500)
 
         @self._app.post('/controller/terminate_replica')

sky/serve/serve_state.py

@@ -13,7 +13,7 @@ from typing import Any, Dict, List, Optional, Tuple
 import colorama
 
 from sky.serve import constants
-from sky.utils import db_utils
+from sky.utils.db import db_utils
 
 if typing.TYPE_CHECKING:
     from sky.serve import replica_managers

sky/serve/serve_utils.py

@@ -12,8 +12,8 @@ import shutil
 import threading
 import time
 import typing
-from typing import (Any, Callable, DefaultDict, Dict, Generic, Iterator, List,
-                    Optional, TextIO, Type, TypeVar, Union)
+from typing import (Any, Callable, DefaultDict, Deque, Dict, Generic, Iterator,
+                    List, Optional, TextIO, Type, TypeVar, Union)
 import uuid
 
 import colorama
@@ -782,6 +782,54 @@ def get_latest_version_with_min_replicas(
     return active_versions[-1] if active_versions else None
 
 
+def _process_line(line: str,
+                  cluster_name: str,
+                  stop_on_eof: bool = False) -> Iterator[str]:
+    # The line might be directing users to view logs, like
+    # `✓ Cluster launched: new-http.  View logs at: *.log`
+    # We should tail the detailed logs for user.
+    def cluster_is_up() -> bool:
+        cluster_record = global_user_state.get_cluster_from_name(cluster_name)
+        if cluster_record is None:
+            return False
+        return cluster_record['status'] == status_lib.ClusterStatus.UP
+
+    provision_log_prompt = re.match(_SKYPILOT_PROVISION_LOG_PATTERN, line)
+    log_prompt = re.match(_SKYPILOT_LOG_PATTERN, line)
+
+    if provision_log_prompt is not None:
+        nested_log_path = os.path.expanduser(provision_log_prompt.group(1))
+
+        try:
+            with open(nested_log_path, 'r', newline='', encoding='utf-8') as f:
+                # We still exit if more than 10 seconds without new content
+                # to avoid any internal bug that causes the launch to fail
+                # while cluster status remains INIT.
+                yield from log_utils.follow_logs(f,
+                                                 should_stop=cluster_is_up,
+                                                 stop_on_eof=stop_on_eof,
+                                                 idle_timeout_seconds=10)
+        except FileNotFoundError:
+            yield line
+
+            yield (f'{colorama.Fore.YELLOW}{colorama.Style.BRIGHT}'
+                   f'Try to expand log file {nested_log_path} but not '
+                   f'found. Skipping...{colorama.Style.RESET_ALL}')
+            pass
+        return
+
+    if log_prompt is not None:
+        # Now we skip other logs (file sync logs) since we lack
+        # utility to determine when these log files are finished
+        # writing.
+        # TODO(tian): We should not skip these logs since there are
+        # small chance that error will happen in file sync. Need to
+        # find a better way to do this.
+        return
+
+    yield line
+
+
 def _follow_logs_with_provision_expanding(
     file: TextIO,
     cluster_name: str,
@@ -804,51 +852,8 @@ def _follow_logs_with_provision_expanding(
         Log lines, including expanded content from referenced provision logs.
     """
 
-    def cluster_is_up() -> bool:
-        cluster_record = global_user_state.get_cluster_from_name(cluster_name)
-        if cluster_record is None:
-            return False
-        return cluster_record['status'] == status_lib.ClusterStatus.UP
-
     def process_line(line: str) -> Iterator[str]:
-        # The line might be directing users to view logs, like
-        # `✓ Cluster launched: new-http.  View logs at: *.log`
-        # We should tail the detailed logs for user.
-        provision_log_prompt = re.match(_SKYPILOT_PROVISION_LOG_PATTERN, line)
-        log_prompt = re.match(_SKYPILOT_LOG_PATTERN, line)
-
-        if provision_log_prompt is not None:
-            nested_log_path = os.path.expanduser(provision_log_prompt.group(1))
-
-            try:
-                with open(nested_log_path, 'r', newline='',
-                          encoding='utf-8') as f:
-                    # We still exit if more than 10 seconds without new content
-                    # to avoid any internal bug that causes the launch to fail
-                    # while cluster status remains INIT.
-                    yield from log_utils.follow_logs(f,
-                                                     should_stop=cluster_is_up,
-                                                     stop_on_eof=stop_on_eof,
-                                                     idle_timeout_seconds=10)
-            except FileNotFoundError:
-                yield line
-
-                yield (f'{colorama.Fore.YELLOW}{colorama.Style.BRIGHT}'
-                       f'Try to expand log file {nested_log_path} but not '
-                       f'found. Skipping...{colorama.Style.RESET_ALL}')
-                pass
-            return
-
-        if log_prompt is not None:
-            # Now we skip other logs (file sync logs) since we lack
-            # utility to determine when these log files are finished
-            # writing.
-            # TODO(tian): We should not skip these logs since there are
-            # small chance that error will happen in file sync. Need to
-            # find a better way to do this.
-            return
-
-        yield line
+        yield from _process_line(line, cluster_name, stop_on_eof=stop_on_eof)
 
     return log_utils.follow_logs(file,
                                  should_stop=should_stop,
@@ -857,18 +862,51 @@ def _follow_logs_with_provision_expanding(
                                  idle_timeout_seconds=idle_timeout_seconds)
 
 
-def stream_replica_logs(service_name: str, replica_id: int,
-                        follow: bool) -> str:
+def _capped_follow_logs_with_provision_expanding(
+    log_list: List[str],
+    cluster_name: str,
+    *,
+    line_cap: int = 100,
+) -> Iterator[str]:
+    """Follows logs and expands any provision.log references found.
+
+    Args:
+        log_list: List of Log Lines to read from.
+        cluster_name: Name of the cluster being launched.
+        line_cap: Number of last lines to return
+
+    Yields:
+        Log lines, including expanded content from referenced provision logs.
+    """
+    all_lines: Deque[str] = collections.deque(maxlen=line_cap)
+
+    for line in log_list:
+        for processed in _process_line(line=line,
+                                       cluster_name=cluster_name,
+                                       stop_on_eof=False):
+            all_lines.append(processed)
+
+    yield from all_lines
+
+
+def stream_replica_logs(service_name: str, replica_id: int, follow: bool,
+                        tail: Optional[int]) -> str:
     msg = check_service_status_healthy(service_name)
     if msg is not None:
         return msg
     print(f'{colorama.Fore.YELLOW}Start streaming logs for launching process '
           f'of replica {replica_id}.{colorama.Style.RESET_ALL}')
-
     log_file_name = generate_replica_log_file_name(service_name, replica_id)
     if os.path.exists(log_file_name):
-        with open(log_file_name, 'r', encoding='utf-8') as f:
-            print(f.read(), flush=True)
+        if tail is not None:
+            lines = common_utils.read_last_n_lines(log_file_name, tail)
+            for line in lines:
+                if not line.endswith('\n'):
+                    line += '\n'
+                print(line, end='', flush=True)
+        else:
+            with open(log_file_name, 'r', encoding='utf-8') as f:
+                print(f.read(), flush=True)
         return ''
 
     launch_log_file_name = generate_replica_launch_log_file_name(
@@ -891,24 +929,48 @@ def stream_replica_logs(service_name: str, replica_id: int,
 
     replica_provisioned = (
         lambda: _get_replica_status() != serve_state.ReplicaStatus.PROVISIONING)
-    with open(launch_log_file_name, 'r', newline='', encoding='utf-8') as f:
-        for line in _follow_logs_with_provision_expanding(
-                f,
-                replica_cluster_name,
-                should_stop=replica_provisioned,
-                stop_on_eof=not follow,
-        ):
-            print(line, end='', flush=True)
+
+    # Handle launch logs based on number parameter
+    final_lines_to_print = []
+    if tail is not None:
+        static_lines = common_utils.read_last_n_lines(launch_log_file_name,
+                                                      tail)
+        lines = list(
+            _capped_follow_logs_with_provision_expanding(
+                log_list=static_lines,
+                cluster_name=replica_cluster_name,
+                line_cap=tail,
+            ))
+        final_lines_to_print += lines
+    else:
+        with open(launch_log_file_name, 'r', newline='', encoding='utf-8') as f:
+            for line in _follow_logs_with_provision_expanding(
+                    f,
+                    replica_cluster_name,
+                    should_stop=replica_provisioned,
+                    stop_on_eof=not follow,
+            ):
+                print(line, end='', flush=True)
 
     if (not follow and
             _get_replica_status() == serve_state.ReplicaStatus.PROVISIONING):
         # Early exit if not following the logs.
+        if tail is not None:
+            for line in final_lines_to_print:
+                if not line.endswith('\n'):
+                    line += '\n'
+                print(line, end='', flush=True)
         return ''
 
     backend = backends.CloudVmRayBackend()
     handle = global_user_state.get_handle_from_cluster_name(
         replica_cluster_name)
     if handle is None:
+        if tail is not None:
+            for line in final_lines_to_print:
+                if not line.endswith('\n'):
+                    line += '\n'
+                print(line, end='', flush=True)
         return _FAILED_TO_FIND_REPLICA_MSG.format(replica_id=replica_id)
     assert isinstance(handle, backends.CloudVmRayResourceHandle), handle
 
@@ -917,15 +979,37 @@ def stream_replica_logs(service_name: str, replica_id: int,
           f'of replica {replica_id}...{colorama.Style.RESET_ALL}')
 
     # Always tail the latest logs, which represent user setup & run.
-    returncode = backend.tail_logs(handle, job_id=None, follow=follow)
-    if returncode != 0:
-        return (f'{colorama.Fore.RED}Failed to stream logs for replica '
-                f'{replica_id}.{colorama.Style.RESET_ALL}')
+    if tail is None:
+        returncode = backend.tail_logs(handle, job_id=None, follow=follow)
+        if returncode != 0:
+            return (f'{colorama.Fore.RED}Failed to stream logs for replica '
+                    f'{replica_id}.{colorama.Style.RESET_ALL}')
+    elif not follow and tail > 0:
+        final = backend.tail_logs(handle,
+                                  job_id=None,
+                                  follow=follow,
+                                  tail=tail,
+                                  stream_logs=False,
+                                  require_outputs=True,
+                                  process_stream=True)
+        if isinstance(final, int) or (final[0] != 0 and final[0] != 101):
+            if tail is not None:
+                for line in final_lines_to_print:
+                    if not line.endswith('\n'):
+                        line += '\n'
+                    print(line, end='', flush=True)
+            return (f'{colorama.Fore.RED}Failed to stream logs for replica '
+                    f'{replica_id}.{colorama.Style.RESET_ALL}')
+        final_lines_to_print += final[1].splitlines()
+        for line in final_lines_to_print[-tail:]:
+            if not line.endswith('\n'):
+                line += '\n'
+            print(line, end='', flush=True)
     return ''
 
 
 def stream_serve_process_logs(service_name: str, stream_controller: bool,
-                              follow: bool) -> str:
+                              follow: bool, tail: Optional[int]) -> str:
     msg = check_service_status_healthy(service_name)
     if msg is not None:
         return msg
@@ -940,14 +1024,24 @@ def stream_serve_process_logs(service_name: str, stream_controller: bool,
             return True
         return record['status'] in serve_state.ServiceStatus.failed_statuses()
 
-    with open(os.path.expanduser(log_file), 'r', newline='',
-              encoding='utf-8') as f:
-        for line in log_utils.follow_logs(
-                f,
-                should_stop=_service_is_terminal,
-                stop_on_eof=not follow,
-        ):
+    if tail is not None:
+        lines = common_utils.read_last_n_lines(os.path.expanduser(log_file),
+                                               tail)
+        for line in lines:
+            if not line.endswith('\n'):
+                line += '\n'
             print(line, end='', flush=True)
+    else:
+        with open(os.path.expanduser(log_file),
+                  'r',
+                  newline='',
+                  encoding='utf-8') as f:
+            for line in log_utils.follow_logs(
+                    f,
+                    should_stop=_service_is_terminal,
+                    stop_on_eof=not follow,
+            ):
+                print(line, end='', flush=True)
     return ''
 
 
@@ -1140,20 +1234,22 @@ class ServeCodeGen:
 
     @classmethod
     def stream_replica_logs(cls, service_name: str, replica_id: int,
-                            follow: bool) -> str:
+                            follow: bool, tail: Optional[int]) -> str:
         code = [
             'msg = serve_utils.stream_replica_logs('
-            f'{service_name!r}, {replica_id!r}, follow={follow})',
+            f'{service_name!r}, {replica_id!r}, follow={follow}, tail={tail})',
             'print(msg, flush=True)'
         ]
         return cls._build(code)
 
     @classmethod
     def stream_serve_process_logs(cls, service_name: str,
-                                  stream_controller: bool, follow: bool) -> str:
+                                  stream_controller: bool, follow: bool,
+                                  tail: Optional[int]) -> str:
         code = [
             f'msg = serve_utils.stream_serve_process_logs({service_name!r}, '
-            f'{stream_controller}, follow={follow})', 'print(msg, flush=True)'
+            f'{stream_controller}, follow={follow}, tail={tail})',
+            'print(msg, flush=True)'
         ]
         return cls._build(code)
 

sky/serve/server/core.py

@@ -740,6 +740,7 @@ def tail_logs(
     target: ServiceComponentOrStr,
     replica_id: Optional[int] = None,
     follow: bool = True,
+    tail: Optional[int] = None,
 ) -> None:
     """Tails logs for a service.
 
@@ -805,11 +806,14 @@ def tail_logs(
             service_name,
             stream_controller=(
                 target == serve_utils.ServiceComponent.CONTROLLER),
-            follow=follow)
+            follow=follow,
+            tail=tail)
     else:
         assert replica_id is not None, service_name
-        code = serve_utils.ServeCodeGen.stream_replica_logs(
-            service_name, replica_id, follow)
+        code = serve_utils.ServeCodeGen.stream_replica_logs(service_name,
+                                                            replica_id,
+                                                            follow,
+                                                            tail=tail)
 
     # With the stdin=subprocess.DEVNULL, the ctrl-c will not directly
     # kill the process, so we need to handle it manually here.
@@ -834,6 +838,7 @@ def sync_down_logs(
     targets: Union[ServiceComponentOrStr, List[ServiceComponentOrStr],
                    None] = None,
     replica_ids: Optional[List[int]] = None,
+    tail: Optional[int] = None,
 ) -> str:
     """Sync down logs from the controller for the given service.
 
@@ -936,16 +941,22 @@ def sync_down_logs(
         if component == serve_utils.ServiceComponent.CONTROLLER:
             stream_logs_code = (
                 serve_utils.ServeCodeGen.stream_serve_process_logs(
-                    service_name, stream_controller=True, follow=False))
+                    service_name,
+                    stream_controller=True,
+                    follow=False,
+                    tail=tail))
         elif component == serve_utils.ServiceComponent.LOAD_BALANCER:
             stream_logs_code = (
                 serve_utils.ServeCodeGen.stream_serve_process_logs(
-                    service_name, stream_controller=False, follow=False))
+                    service_name,
+                    stream_controller=False,
+                    follow=False,
+                    tail=tail))
         elif component == serve_utils.ServiceComponent.REPLICA:
             replica_id = target.replica_id
             assert replica_id is not None, service_name
             stream_logs_code = serve_utils.ServeCodeGen.stream_replica_logs(
-                service_name, replica_id, follow=False)
+                service_name, replica_id, follow=False, tail=tail)
         else:
             assert False, component
 

sky/server/common.py

@@ -371,6 +371,10 @@ def get_api_server_status(endpoint: Optional[str] = None) -> ApiServerInfo:
 
 
 def handle_request_error(response: 'requests.Response') -> None:
+    # Keep the original HTTPError if the response code >= 400
+    response.raise_for_status()
+    # Other status codes are not expected neither, e.g. we do not expect to
+    # handle redirection here.
     if response.status_code != 200:
         with ux_utils.print_exception_no_traceback():
             raise RuntimeError(

sky/server/requests/payloads.py

@@ -557,6 +557,7 @@ class ServeLogsBody(RequestBody):
     target: Union[str, serve.ServiceComponent]
     replica_id: Optional[int] = None
     follow: bool = True
+    tail: Optional[int] = None
 
 
 class ServeDownloadLogsBody(RequestBody):
@@ -566,6 +567,7 @@ class ServeDownloadLogsBody(RequestBody):
     targets: Optional[Union[str, serve.ServiceComponent,
                             List[Union[str, serve.ServiceComponent]]]]
     replica_ids: Optional[List[int]] = None
+    tail: Optional[int] = None
 
 
 class ServeStatusBody(RequestBody):

sky/server/requests/requests.py

@@ -29,10 +29,10 @@ from sky.server.requests.serializers import decoders
 from sky.server.requests.serializers import encoders
 from sky.utils import common
 from sky.utils import common_utils
-from sky.utils import db_utils
 from sky.utils import env_options
 from sky.utils import subprocess_utils
 from sky.utils import ux_utils
+from sky.utils.db import db_utils
 
 logger = sky_logging.init_logger(__name__)