skypilot-nightly 1.0.0.dev20250509__py3-none-any.whl → 1.0.0.dev20251107__py3-none-any.whl

sky/schemas/api/responses.py

@@ -0,0 +1,225 @@
+"""Responses for the API server."""
+
+import enum
+from typing import Any, Dict, List, Optional
+
+import pydantic
+
+from sky import data
+from sky import models
+from sky.jobs import state as job_state
+from sky.server import common
+from sky.skylet import job_lib
+from sky.utils import status_lib
+
+
+class ResponseBaseModel(pydantic.BaseModel):
+    """A pydantic model that acts like a dict.
+
+    Supports the following syntax:
+    class SampleResponse(DictLikePayload):
+        field: str
+
+    response = SampleResponse(field='value')
+    print(response['field']) # prints 'value'
+    response['field'] = 'value2'
+    print(response['field']) # prints 'value2'
+    print('field' in response) # prints True
+
+    This model exists for backwards compatibility with the
+    old SDK that used to return a dict.
+
+    The backward compatibility may be removed
+    in the future.
+    """
+    # Ignore extra fields in the request body, which is useful for backward
+    # compatibility. The difference with `allow` is that `ignore` will not
+    # include the unknown fields when dump the model, i.e., we can add new
+    # fields to the request body without breaking the existing old API server
+    # where the handler function does not accept the new field in function
+    # signature.
+    model_config = pydantic.ConfigDict(extra='ignore')
+
+    # backward compatibility with dict
+    # TODO(syang): remove this in v0.13.0
+    def __getitem__(self, key):
+        try:
+            return getattr(self, key)
+        except AttributeError as e:
+            raise KeyError(key) from e
+
+    def __setitem__(self, key, value):
+        setattr(self, key, value)
+
+    def get(self, key, default=None):
+        return getattr(self, key, default)
+
+    def __contains__(self, key):
+        return hasattr(self, key)
+
+    def keys(self):
+        return self.model_dump().keys()
+
+    def values(self):
+        return self.model_dump().values()
+
+    def items(self):
+        return self.model_dump().items()
+
+    def __repr__(self):
+        return self.__dict__.__repr__()
+
+
+class APIHealthResponse(ResponseBaseModel):
+    """Response for the API health endpoint."""
+    status: common.ApiServerStatus
+    api_version: str = ''
+    version: str = ''
+    version_on_disk: str = ''
+    commit: str = ''
+    # Whether basic auth on api server is enabled
+    basic_auth_enabled: bool = False
+    user: Optional[models.User] = None
+    # Whether service account token is enabled
+    service_account_token_enabled: bool = False
+    # Whether basic auth on ingress is enabled
+    ingress_basic_auth_enabled: bool = False
+
+
+class StatusResponse(ResponseBaseModel):
+    """Response for the status endpoint."""
+    name: str
+    launched_at: int
+    # pydantic cannot generate the pydantic-core schema for
+    # backends.ResourceHandle, so we use Any here.
+    # This is an internally facing field anyway, so it's less
+    # of a problem that it's not typed.
+    handle: Optional[Any] = None
+    last_use: Optional[str] = None
+    status: status_lib.ClusterStatus
+    autostop: int
+    to_down: bool
+    owner: Optional[List[str]] = None
+    # metadata is a JSON, so we use Any here.
+    metadata: Optional[Dict[str, Any]] = None
+    cluster_hash: str
+    cluster_ever_up: bool
+    status_updated_at: Optional[int] = None
+    user_hash: str
+    user_name: str
+    config_hash: Optional[str] = None
+    workspace: str
+    last_creation_yaml: Optional[str] = None
+    last_creation_command: Optional[str] = None
+    is_managed: bool
+    last_event: Optional[str] = None
+    resources_str: Optional[str] = None
+    resources_str_full: Optional[str] = None
+    # credentials is a JSON, so we use Any here.
+    credentials: Optional[Dict[str, Any]] = None
+    nodes: int
+    cloud: Optional[str] = None
+    region: Optional[str] = None
+    cpus: Optional[str] = None
+    memory: Optional[str] = None
+    accelerators: Optional[str] = None
+    cluster_name_on_cloud: Optional[str] = None
+
+
+class ClusterJobRecord(ResponseBaseModel):
+    """Response for the cluster job queue endpoint."""
+    job_id: int
+    job_name: str
+    username: str
+    user_hash: str
+    submitted_at: float
+    # None if the job has not started yet.
+    start_at: Optional[float] = None
+    # None if the job has not ended yet.
+    end_at: Optional[float] = None
+    resources: str
+    status: job_lib.JobStatus
+    log_path: str
+    metadata: Dict[str, Any] = {}
+
+
+class UploadStatus(enum.Enum):
+    """Status of the upload."""
+    UPLOADING = 'uploading'
+    COMPLETED = 'completed'
+
+
+class StorageRecord(ResponseBaseModel):
+    """Response for the storage list endpoint."""
+    name: str
+    launched_at: int
+    store: List[data.StoreType]
+    last_use: str
+    status: status_lib.StorageStatus
+
+
+# TODO (syang) figure out which fields are always present
+# and therefore can be non-optional.
+class ManagedJobRecord(ResponseBaseModel):
+    """A single managed job record."""
+    # The job_id in the spot table
+    task_job_id: Optional[int] = pydantic.Field(None, alias='_job_id')
+    job_id: Optional[int] = None
+    task_id: Optional[int] = None
+    job_name: Optional[str] = None
+    task_name: Optional[str] = None
+    job_duration: Optional[float] = None
+    workspace: Optional[str] = None
+    status: Optional[job_state.ManagedJobStatus] = None
+    schedule_state: Optional[str] = None
+    resources: Optional[str] = None
+    cluster_resources: Optional[str] = None
+    cluster_resources_full: Optional[str] = None
+    cloud: Optional[str] = None
+    region: Optional[str] = None
+    zone: Optional[str] = None
+    infra: Optional[str] = None
+    recovery_count: Optional[int] = None
+    details: Optional[str] = None
+    failure_reason: Optional[str] = None
+    user_name: Optional[str] = None
+    user_hash: Optional[str] = None
+    submitted_at: Optional[float] = None
+    start_at: Optional[float] = None
+    end_at: Optional[float] = None
+    user_yaml: Optional[str] = None
+    entrypoint: Optional[str] = None
+    metadata: Optional[Dict[str, Any]] = None
+    controller_pid: Optional[int] = None
+    dag_yaml_path: Optional[str] = None
+    env_file_path: Optional[str] = None
+    last_recovered_at: Optional[float] = None
+    run_timestamp: Optional[str] = None
+    priority: Optional[int] = None
+    original_user_yaml_path: Optional[str] = None
+    pool: Optional[str] = None
+    pool_hash: Optional[str] = None
+    current_cluster_name: Optional[str] = None
+    job_id_on_pool_cluster: Optional[int] = None
+    accelerators: Optional[Dict[str, int]] = None
+
+
+class VolumeRecord(ResponseBaseModel):
+    """A single volume record."""
+    name: str
+    type: str
+    launched_at: int
+    cloud: str
+    region: Optional[str] = None
+    zone: Optional[str] = None
+    size: Optional[str] = None
+    config: Dict[str, Any]
+    name_on_cloud: str
+    user_hash: str
+    user_name: str
+    workspace: str
+    last_attached_at: Optional[int] = None
+    last_use: Optional[str] = None
+    status: Optional[str] = None
+    usedby_pods: List[str]
+    usedby_clusters: List[str]

sky/schemas/db/README

@@ -0,0 +1,4 @@
+Migrations for sqlalchemy databases. Currently includes:
+global_user_state
+spot_jobs (managed jobs state)
+skypilot_config

sky/schemas/db/env.py

@@ -0,0 +1,90 @@
+"""Alembic environment configuration for state database migrations."""
+from logging.config import fileConfig
+
+from alembic import context
+from sqlalchemy import engine_from_config
+from sqlalchemy import pool
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# NOTE: We intentionally disable Alembic's logging configuration to prevent
+# it from overriding SkyPilot's logging setup. Alembic's fileConfig() call
+# globally reconfigures Python's logging system, which can suppress SkyPilot's
+# output messages that tests expect to see.
+#
+# Original code (now disabled):
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name, disable_existing_loggers=False)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = None
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option('sqlalchemy.url')
+    version_table = config.get_section_option(config.config_ini_section,
+                                              'version_table',
+                                              'alembic_version')
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={'paramstyle': 'named'},
+        version_table=version_table,
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix='sqlalchemy.',
+        poolclass=pool.NullPool,
+    )
+    version_table = config.get_section_option(config.config_ini_section,
+                                              'version_table',
+                                              'alembic_version')
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            version_table=version_table,
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

sky/schemas/db/global_user_state/001_initial_schema.py

@@ -0,0 +1,124 @@
+"""Initial schema for state database with backwards compatibility columns
+
+Revision ID: 001
+Revises:
+Create Date: 2024-01-01 12:00:00.000000
+
+"""
+# pylint: disable=invalid-name
+from alembic import op
+import sqlalchemy as sa
+
+from sky.global_user_state import Base
+from sky.utils.db import db_utils
+
+# revision identifiers, used by Alembic.
+revision = '001'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.get_context().autocommit_block():
+        # Create any missing tables with current schema first
+        db_utils.add_all_tables_to_db_sqlalchemy(Base.metadata, op.get_bind())
+
+        # Add all missing columns to clusters table
+        # This allows each column addition to fail independently without rolling
+        # back the entire migration, which is needed for backwards compatibility
+
+        # Add all missing columns to clusters table
+        db_utils.add_column_to_table_alembic('clusters',
+                                             'autostop',
+                                             sa.Integer(),
+                                             server_default='-1')
+        db_utils.add_column_to_table_alembic('clusters',
+                                             'metadata',
+                                             sa.Text(),
+                                             server_default='{}')
+        db_utils.add_column_to_table_alembic('clusters',
+                                             'to_down',
+                                             sa.Integer(),
+                                             server_default='0')
+        db_utils.add_column_to_table_alembic('clusters',
+                                             'owner',
+                                             sa.Text(),
+                                             server_default=None)
+        db_utils.add_column_to_table_alembic('clusters',
+                                             'cluster_hash',
+                                             sa.Text(),
+                                             server_default=None)
+        db_utils.add_column_to_table_alembic('clusters',
+                                             'launched_nodes',
+                                             sa.Integer(),
+                                             server_default='0')
+        db_utils.add_column_to_table_alembic('clusters',
+                                             'disk_tier',
+                                             sa.Text(),
+                                             server_default=None)
+        db_utils.add_column_to_table_alembic('clusters',
+                                             'config_hash',
+                                             sa.Text(),
+                                             server_default=None)
+        db_utils.add_column_to_table_alembic('clusters',
+                                             'user_hash',
+                                             sa.Text(),
+                                             server_default=None)
+        db_utils.add_column_to_table_alembic('clusters',
+                                             'workspace',
+                                             sa.Text(),
+                                             server_default='default')
+        db_utils.add_column_to_table_alembic('clusters',
+                                             'last_creation_yaml',
+                                             sa.Text(),
+                                             server_default=None)
+        db_utils.add_column_to_table_alembic('clusters',
+                                             'last_creation_command',
+                                             sa.Text(),
+                                             server_default=None)
+        db_utils.add_column_to_table_alembic('clusters',
+                                             'config_hash_locked',
+                                             sa.Boolean(),
+                                             server_default='FALSE')
+        db_utils.add_column_to_table_alembic('clusters',
+                                             'handle_locked',
+                                             sa.Boolean(),
+                                             server_default='FALSE')
+        db_utils.add_column_to_table_alembic('clusters',
+                                             'num_failures',
+                                             sa.Integer(),
+                                             server_default='0')
+        db_utils.add_column_to_table_alembic('clusters',
+                                             'configs',
+                                             sa.Text(),
+                                             server_default='[]')
+
+        # Add all missing columns to cluster_history table
+        db_utils.add_column_to_table_alembic('cluster_history',
+                                             'user_hash',
+                                             sa.Text(),
+                                             server_default=None)
+        db_utils.add_column_to_table_alembic('cluster_history',
+                                             'last_creation_yaml',
+                                             sa.Text(),
+                                             server_default=None)
+        db_utils.add_column_to_table_alembic('cluster_history',
+                                             'last_creation_command',
+                                             sa.Text(),
+                                             server_default=None)
+
+        # Add all missing columns to users table
+        db_utils.add_column_to_table_alembic('users',
+                                             'password',
+                                             sa.Text(),
+                                             server_default=None)
+        db_utils.add_column_to_table_alembic('users',
+                                             'created_at',
+                                             sa.Integer(),
+                                             server_default=None)
+
+
+def downgrade():
+    # Drop all tables
+    Base.metadata.drop_all(bind=op.get_bind())

sky/schemas/db/global_user_state/002_add_workspace_to_cluster_history.py

@@ -0,0 +1,35 @@
+"""add workspace column to cluster_history table
+
+Revision ID: 002
+Revises: 001
+Create Date: 2025-08-06
+
+"""
+# pylint: disable=invalid-name
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+from sky.utils.db import db_utils
+
+# revision identifiers, used by Alembic.
+revision: str = '002'
+down_revision: Union[str, Sequence[str], None] = '001'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    with op.get_context().autocommit_block():
+        db_utils.add_column_to_table_alembic('cluster_history',
+                                             'workspace',
+                                             sa.Text(),
+                                             server_default=None)
+    pass
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    pass

sky/schemas/db/global_user_state/003_fix_initial_revision.py

@@ -0,0 +1,61 @@
+"""fix initial revision
+
+Revision ID: 003
+Revises: 002
+Create Date: 2025-08-07
+
+"""
+# pylint: disable=invalid-name
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+from sky.utils.db import db_utils
+
+# revision identifiers, used by Alembic.
+revision: str = '003'
+down_revision: Union[str, Sequence[str], None] = '002'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    with op.get_context().autocommit_block():
+        # add missing columns to clusters table
+        db_utils.add_column_to_table_alembic('clusters',
+                                             'storage_mounts_metadata',
+                                             sa.LargeBinary(),
+                                             server_default=None)
+        # Set the value to replace existing entries to 1 so that all the
+        # existing clusters before #2977 are considered as ever up, i.e:
+        #   existing cluster's default (null) -> 1;
+        #   new cluster's default -> 0;
+        # This is conservative for the existing clusters: even if some INIT
+        # clusters were never really UP, setting it to 1 means they won't be
+        # auto-deleted during any failover.
+        db_utils.add_column_to_table_alembic(
+            'clusters',
+            'cluster_ever_up',
+            sa.Integer(),
+            server_default='0',
+            value_to_replace_existing_entries=1)
+        db_utils.add_column_to_table_alembic('clusters',
+                                             'status_updated_at',
+                                             sa.Integer(),
+                                             server_default=None)
+
+        # remove mistakenly added columns
+        db_utils.drop_column_from_table_alembic('clusters', 'launched_nodes')
+        db_utils.drop_column_from_table_alembic('clusters', 'disk_tier')
+        db_utils.drop_column_from_table_alembic('clusters',
+                                                'config_hash_locked')
+        db_utils.drop_column_from_table_alembic('clusters', 'handle_locked')
+        db_utils.drop_column_from_table_alembic('clusters', 'num_failures')
+        db_utils.drop_column_from_table_alembic('clusters', 'configs')
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    pass

sky/schemas/db/global_user_state/004_is_managed.py

@@ -0,0 +1,34 @@
+"""Columns for whether the cluster is managed.
+
+Revision ID: 004
+Revises: 003
+Create Date: 2025-08-07
+
+"""
+# pylint: disable=invalid-name
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+from sky.utils.db import db_utils
+
+# revision identifiers, used by Alembic.π
+revision: str = '004'
+down_revision: Union[str, Sequence[str], None] = '003'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade():
+    """Add columns for whether the cluster is managed."""
+    with op.get_context().autocommit_block():
+        db_utils.add_column_to_table_alembic('clusters',
+                                             'is_managed',
+                                             sa.Integer(),
+                                             server_default='0')
+
+
+def downgrade():
+    """Remove columns for whether the cluster is managed."""
+    pass

sky/schemas/db/global_user_state/005_cluster_event.py

@@ -0,0 +1,32 @@
+"""Columns for whether the cluster is managed.
+
+Revision ID: 005
+Revises: 004
+Create Date: 2025-08-08
+
+"""
+# pylint: disable=invalid-name
+from typing import Sequence, Union
+
+from alembic import op
+
+from sky.global_user_state import Base
+from sky.utils.db import db_utils
+
+# revision identifiers, used by Alembic.π
+revision: str = '005'
+down_revision: Union[str, Sequence[str], None] = '004'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade():
+    """Add new table for cluster events."""
+    with op.get_context().autocommit_block():
+        # Add new table for cluster events.
+        db_utils.add_table_to_db_sqlalchemy(Base.metadata, op.get_bind(),
+                                            'cluster_events')
+
+
+def downgrade():
+    pass

sky/schemas/db/global_user_state/006_provision_log.py

@@ -0,0 +1,41 @@
+"""Add provision_log_path to clusters and cluster_history.
+
+Revision ID: 006
+Revises: 005
+Create Date: 2025-08-12
+
+"""
+# pylint: disable=invalid-name
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+from sky.utils.db import db_utils
+
+# revision identifiers, used by Alembic.
+revision: str = '006'
+down_revision: Union[str, Sequence[str], None] = '005'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade():
+    """Add provision_log_path columns."""
+    with op.get_context().autocommit_block():
+        # clusters.provision_log_path
+        db_utils.add_column_to_table_alembic('clusters',
+                                             'provision_log_path',
+                                             sa.Text(),
+                                             server_default=None)
+
+        # cluster_history.provision_log_path
+        db_utils.add_column_to_table_alembic('cluster_history',
+                                             'provision_log_path',
+                                             sa.Text(),
+                                             server_default=None)
+
+
+def downgrade():
+    """No-op for backward compatibility."""
+    pass

sky/schemas/db/global_user_state/007_cluster_event_request_id.py

@@ -0,0 +1,34 @@
+"""Add request_id to cluster_events.
+
+Revision ID: 007
+Revises: 006
+Create Date: 2025-08-28
+
+"""
+# pylint: disable=invalid-name
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+from sky.utils.db import db_utils
+
+# revision identifiers, used by Alembic.
+revision: str = '007'
+down_revision: Union[str, Sequence[str], None] = '006'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade():
+    """Add request_id column to cluster_events."""
+    with op.get_context().autocommit_block():
+        db_utils.add_column_to_table_alembic('cluster_events',
+                                             'request_id',
+                                             sa.Text(),
+                                             server_default=None)
+
+
+def downgrade():
+    """No-op for backward compatibility."""
+    pass