dstack 0.18.44__py3-none-any.whl → 0.19.0__py3-none-any.whl

dstack/_internal/core/services/repos.py

@@ -10,7 +10,6 @@ from git.exc import GitCommandError
 from dstack._internal.core.errors import DstackError
 from dstack._internal.core.models.config import RepoConfig
 from dstack._internal.core.models.repos import LocalRepo, RemoteRepo, RemoteRepoCreds
-from dstack._internal.core.models.repos.base import RepoProtocol
 from dstack._internal.core.models.repos.remote import GitRepoURL
 from dstack._internal.utils.logging import get_logger
 from dstack._internal.utils.path import PathLike
@@ -41,7 +40,6 @@ def get_local_repo_credentials(
     r = requests.get(f"{url.as_https()}/info/refs?service=git-upload-pack", timeout=10)
     if r.status_code == 200:
         return RemoteRepoCreds(
-            protocol=RepoProtocol.HTTPS,
             clone_url=url.as_https(),
             private_key=None,
             oauth_token=None,
@@ -93,7 +91,6 @@ def check_remote_repo_credentials_https(url: GitRepoURL, oauth_token: str) -> Re
             f"Can't access `{url.as_https()}` using the `{masked}` token"
         )
     return RemoteRepoCreds(
-        protocol=RepoProtocol.HTTPS,
         clone_url=url.as_https(),
         oauth_token=oauth_token,
         private_key=None,
@@ -123,7 +120,6 @@ def check_remote_repo_credentials_ssh(url: GitRepoURL, identity_file: PathLike)
         )
 
     return RemoteRepoCreds(
-        protocol=RepoProtocol.SSH,
         clone_url=url.as_ssh(),
         private_key=private_key,
         oauth_token=None,

dstack/_internal/server/app.py

@@ -27,7 +27,6 @@ from dstack._internal.server.routers import (
     instances,
     logs,
     metrics,
-    pools,
     projects,
     prometheus,
     repos,
@@ -184,8 +183,6 @@ def register_routes(app: FastAPI, ui: bool = True):
     app.include_router(volumes.project_router)
     app.include_router(service_proxy.router, prefix="/proxy/services", tags=["service-proxy"])
     app.include_router(model_proxy.router, prefix="/proxy/models", tags=["model-proxy"])
-    app.include_router(pools.root_router)
-    app.include_router(pools.router)
     app.include_router(prometheus.router)
 
     @app.exception_handler(ForbiddenError)

dstack/_internal/server/background/tasks/process_gateways.py

@@ -54,17 +54,13 @@ async def process_submitted_gateways():
 
 
 async def _remove_inactive_connections():
-    connections = await gateway_connections_pool.all()
-    ip_addresses = [c.ip_address for c in connections]
     async with get_session_ctx() as session:
         res = await session.execute(
-            select(GatewayComputeModel).where(
-                GatewayComputeModel.ip_address.in_(ip_addresses),
-                GatewayComputeModel.active == False,
-            )
+            select(GatewayComputeModel.ip_address).where(GatewayComputeModel.active == True)
         )
-        removed_connections = res.scalars().all()
-        for conn in removed_connections:
+    active_connection_ips = set(res.scalars().all())
+    for conn in await gateway_connections_pool.all():
+        if conn.ip_address not in active_connection_ips:
             await gateway_connections_pool.remove(conn.ip_address)
 
 

dstack/_internal/server/background/tasks/process_instances.py

@@ -20,6 +20,8 @@ from dstack._internal.core.backends.base.compute import (
     DSTACK_RUNNER_BINARY_PATH,
     DSTACK_SHIM_BINARY_PATH,
     DSTACK_WORKING_DIR,
+    ComputeWithCreateInstanceSupport,
+    ComputeWithPlacementGroupSupport,
     get_shim_env,
     get_shim_pre_start_commands,
 )
@@ -76,19 +78,19 @@ from dstack._internal.server.services.fleets import (
     fleet_model_to_fleet,
     get_create_instance_offers,
 )
-from dstack._internal.server.services.locking import get_locker
-from dstack._internal.server.services.offers import is_divisible_into_blocks
-from dstack._internal.server.services.placement import (
-    get_fleet_placement_groups,
-    placement_group_model_to_placement_group,
-)
-from dstack._internal.server.services.pools import (
+from dstack._internal.server.services.instances import (
     get_instance_configuration,
     get_instance_profile,
     get_instance_provisioning_data,
     get_instance_requirements,
     get_instance_ssh_private_keys,
 )
+from dstack._internal.server.services.locking import get_locker
+from dstack._internal.server.services.offers import is_divisible_into_blocks
+from dstack._internal.server.services.placement import (
+    get_fleet_placement_groups,
+    placement_group_model_to_placement_group,
+)
 from dstack._internal.server.services.runner import client as runner_client
 from dstack._internal.server.services.runner.client import HealthStatus
 from dstack._internal.server.services.runner.ssh import runner_ssh_tunnel
@@ -530,12 +532,15 @@ async def _create_instance(session: AsyncSession, instance: InstanceModel) -> No
     for backend, instance_offer in offers:
         if instance_offer.backend not in BACKENDS_WITH_CREATE_INSTANCE_SUPPORT:
             continue
+        compute = backend.compute()
+        assert isinstance(compute, ComputeWithCreateInstanceSupport)
         instance_offer = _get_instance_offer_for_instance(instance_offer, instance)
         if (
             instance_offer.backend in BACKENDS_WITH_PLACEMENT_GROUPS_SUPPORT
             and instance.fleet
             and instance_configuration.placement_group_name
         ):
+            assert isinstance(compute, ComputeWithPlacementGroupSupport)
             placement_group_model = _create_placement_group_if_does_not_exist(
                 session=session,
                 fleet_model=instance.fleet,
@@ -546,7 +551,7 @@ async def _create_instance(session: AsyncSession, instance: InstanceModel) -> No
             )
             if placement_group_model is not None:
                 placement_group = placement_group_model_to_placement_group(placement_group_model)
-                pgpd = await run_async(backend.compute().create_placement_group, placement_group)
+                pgpd = await run_async(compute.create_placement_group, placement_group)
                 placement_group_model.provisioning_data = pgpd.json()
                 session.add(placement_group_model)
                 placement_groups.append(placement_group)
@@ -559,7 +564,7 @@ async def _create_instance(session: AsyncSession, instance: InstanceModel) -> No
         )
         try:
             job_provisioning_data = await run_async(
-                backend.compute().create_instance,
+                compute.create_instance,
                 instance_offer,
                 instance_configuration,
             )

dstack/_internal/server/background/tasks/process_metrics.py

@@ -11,8 +11,8 @@ from dstack._internal.server import settings
 from dstack._internal.server.db import get_session_ctx
 from dstack._internal.server.models import InstanceModel, JobMetricsPoint, JobModel
 from dstack._internal.server.schemas.runner import MetricsResponse
+from dstack._internal.server.services.instances import get_instance_ssh_private_keys
 from dstack._internal.server.services.jobs import get_job_provisioning_data, get_job_runtime_data
-from dstack._internal.server.services.pools import get_instance_ssh_private_keys
 from dstack._internal.server.services.runner import client
 from dstack._internal.server.services.runner.ssh import runner_ssh_tunnel
 from dstack._internal.utils.common import batched, get_current_datetime, get_or_error, run_async

dstack/_internal/server/background/tasks/process_placement_groups.py

@@ -5,6 +5,7 @@ from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import joinedload
 
+from dstack._internal.core.backends.base.compute import ComputeWithPlacementGroupSupport
 from dstack._internal.core.errors import PlacementGroupInUseError
 from dstack._internal.server.db import get_session_ctx
 from dstack._internal.server.models import PlacementGroupModel, ProjectModel
@@ -81,8 +82,10 @@ async def _delete_placement_group(placement_group_model: PlacementGroupModel):
             "Failed to delete placement group %s. Backend not available.", placement_group.name
         )
         return
+    compute = backend.compute()
+    assert isinstance(compute, ComputeWithPlacementGroupSupport)
     try:
-        await run_async(backend.compute().delete_placement_group, placement_group)
+        await run_async(compute.delete_placement_group, placement_group)
     except PlacementGroupInUseError:
         logger.info(
             "Placement group %s is still in use. Skipping deletion for now.", placement_group.name

dstack/_internal/server/background/tasks/process_prometheus_metrics.py

@@ -10,8 +10,8 @@ from dstack._internal.core.consts import DSTACK_SHIM_HTTP_PORT
 from dstack._internal.core.models.runs import JobStatus
 from dstack._internal.server.db import get_session_ctx
 from dstack._internal.server.models import InstanceModel, JobModel, JobPrometheusMetrics
+from dstack._internal.server.services.instances import get_instance_ssh_private_keys
 from dstack._internal.server.services.jobs import get_job_provisioning_data, get_job_runtime_data
-from dstack._internal.server.services.pools import get_instance_ssh_private_keys
 from dstack._internal.server.services.runner import client
 from dstack._internal.server.services.runner.ssh import runner_ssh_tunnel
 from dstack._internal.server.utils.common import gather_map_async

dstack/_internal/server/background/tasks/process_running_jobs.py

@@ -43,6 +43,7 @@ from dstack._internal.server.models import (
 from dstack._internal.server.schemas.runner import TaskStatus
 from dstack._internal.server.services import logs as logs_services
 from dstack._internal.server.services import services
+from dstack._internal.server.services.instances import get_instance_ssh_private_keys
 from dstack._internal.server.services.jobs import (
     find_job,
     get_job_attached_volumes,
@@ -52,7 +53,6 @@ from dstack._internal.server.services.jobs import (
 from dstack._internal.server.services.locking import get_locker
 from dstack._internal.server.services.logging import fmt
 from dstack._internal.server.services.metrics import get_job_metrics
-from dstack._internal.server.services.pools import get_instance_ssh_private_keys
 from dstack._internal.server.services.repos import (
     get_code_model,
     get_repo_creds,
@@ -127,7 +127,7 @@ async def _process_running_job(session: AsyncSession, job_model: JobModel):
     run_model = res.unique().scalar_one()
     repo_model = run_model.repo
     project = run_model.project
-    run = run_model_to_run(run_model)
+    run = run_model_to_run(run_model, include_sensitive=True)
     job_submission = job_model_to_job_submission(job_model)
     job_provisioning_data = job_submission.job_provisioning_data
     if job_provisioning_data is None:
@@ -743,20 +743,29 @@ def _get_cluster_info(
 
 
 async def _get_job_code(
-    session: AsyncSession, project: ProjectModel, repo: RepoModel, code_hash: str
+    session: AsyncSession, project: ProjectModel, repo: RepoModel, code_hash: Optional[str]
 ) -> bytes:
+    if code_hash is None:
+        return b""
     code_model = await get_code_model(session=session, repo=repo, code_hash=code_hash)
     if code_model is None:
         return b""
-    storage = get_default_storage()
-    if storage is None or code_model.blob is not None:
+    if code_model.blob is not None:
         return code_model.blob
+    storage = get_default_storage()
+    if storage is None:
+        return b""
     blob = await common_utils.run_async(
         storage.get_code,
         project.name,
         repo.name,
         code_hash,
     )
+    if blob is None:
+        logger.error(
+            "Failed to get repo code hash %s from storage for repo %s", code_hash, repo.name
+        )
+        return b""
     return blob
 
 

dstack/_internal/server/background/tasks/process_submitted_jobs.py

@@ -6,7 +6,8 @@ from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import joinedload, lazyload, selectinload
 
-from dstack._internal.core.backends.base import Backend
+from dstack._internal.core.backends.base.backend import Backend
+from dstack._internal.core.backends.base.compute import ComputeWithVolumeSupport
 from dstack._internal.core.errors import BackendError, ServerClientError
 from dstack._internal.core.models.common import NetworkMode
 from dstack._internal.core.models.fleets import (
@@ -17,10 +18,8 @@ from dstack._internal.core.models.fleets import (
 )
 from dstack._internal.core.models.instances import InstanceOfferWithAvailability, InstanceStatus
 from dstack._internal.core.models.profiles import (
-    DEFAULT_POOL_NAME,
     DEFAULT_RUN_TERMINATION_IDLE_TIME,
     CreationPolicy,
-    Profile,
     TerminationPolicy,
 )
 from dstack._internal.core.models.resources import Memory
@@ -41,7 +40,6 @@ from dstack._internal.server.models import (
     FleetModel,
     InstanceModel,
     JobModel,
-    PoolModel,
     ProjectModel,
     RunModel,
     VolumeAttachmentModel,
@@ -51,6 +49,12 @@ from dstack._internal.server.services.backends import get_project_backend_by_typ
 from dstack._internal.server.services.fleets import (
     fleet_model_to_fleet,
 )
+from dstack._internal.server.services.instances import (
+    filter_pool_instances,
+    get_instance_offer,
+    get_instance_provisioning_data,
+    get_shared_pool_instances_with_offers,
+)
 from dstack._internal.server.services.jobs import (
     check_can_attach_job_volumes,
     find_job,
@@ -62,12 +66,6 @@ from dstack._internal.server.services.jobs import (
 from dstack._internal.server.services.locking import get_locker
 from dstack._internal.server.services.logging import fmt
 from dstack._internal.server.services.offers import get_offers_by_requirements
-from dstack._internal.server.services.pools import (
-    filter_pool_instances,
-    get_instance_offer,
-    get_instance_provisioning_data,
-    get_shared_pool_instances_with_offers,
-)
 from dstack._internal.server.services.runs import (
     check_run_spec_requires_instance_mounts,
     run_model_to_run,
@@ -173,25 +171,22 @@ async def _process_submitted_job(session: AsyncSession, job_model: JobModel):
     except ServerClientError as e:
         logger.warning("%s: failed to prepare run volumes: %s", fmt(job_model), repr(e))
         job_model.status = JobStatus.TERMINATING
-        # TODO: Replace with JobTerminationReason.VOLUME_ERROR in 0.19
-        job_model.termination_reason = JobTerminationReason.TERMINATED_BY_SERVER
+        job_model.termination_reason = JobTerminationReason.VOLUME_ERROR
         job_model.termination_reason_message = e.msg
         job_model.last_processed_at = common_utils.get_current_datetime()
         await session.commit()
         return
 
-    pool = await _get_pool(session=session, project=project, profile=profile)
-
     # Submitted jobs processing happens in two steps (transactions).
     # First, the jobs gets an instance assigned (or no instance).
     # Then, the job runs on the assigned instance or a new instance is provisioned.
     # This is needed to avoid holding instances lock for a long time.
     if not job_model.instance_assigned:
-        # Try assigning instances from the pool.
+        # Try assigning an existing instance
         res = await session.execute(
             select(InstanceModel)
             .where(
-                InstanceModel.pool_id == pool.id,
+                InstanceModel.project_id == project.id,
                 InstanceModel.deleted == False,
                 InstanceModel.total_blocks > InstanceModel.busy_blocks,
             )
@@ -289,7 +284,6 @@ async def _process_submitted_job(session: AsyncSession, job_model: JobModel):
         )
         instance = _create_instance_model_for_job(
             project=project,
-            pool=pool,
             fleet_model=fleet_model,
             run_spec=run_spec,
             job_model=job_model,
@@ -337,19 +331,6 @@ async def _process_submitted_job(session: AsyncSession, job_model: JobModel):
         await session.commit()
 
 
-async def _get_pool(session: AsyncSession, project: ProjectModel, profile: Profile) -> PoolModel:
-    res = await session.execute(
-        select(PoolModel)
-        .where(
-            PoolModel.project_id == project.id,
-            PoolModel.name == (profile.pool_name or DEFAULT_POOL_NAME),
-            PoolModel.deleted == False,
-        )
-        .options(lazyload(PoolModel.instances))
-    )
-    return res.scalar_one()
-
-
 async def _assign_job_to_pool_instance(
     session: AsyncSession,
     pool_instances: List[InstanceModel],
@@ -548,7 +529,6 @@ async def _get_next_instance_num(session: AsyncSession, fleet_model: FleetModel)
 
 def _create_instance_model_for_job(
     project: ProjectModel,
-    pool: PoolModel,
     fleet_model: FleetModel,
     run_spec: RunSpec,
     job_model: JobModel,
@@ -571,7 +551,6 @@ def _create_instance_model_for_job(
         name=f"{fleet_model.name}-{instance_num}",
         instance_num=instance_num,
         project=project,
-        pool=pool,
         created_at=common_utils.get_current_datetime(),
         started_at=common_utils.get_current_datetime(),
         status=InstanceStatus.PROVISIONING,
@@ -677,8 +656,7 @@ async def _attach_volumes(
             except (ServerClientError, BackendError) as e:
                 logger.warning("%s: failed to attached volume: %s", fmt(job_model), repr(e))
                 job_model.status = JobStatus.TERMINATING
-                # TODO: Replace with JobTerminationReason.VOLUME_ERROR in 0.19
-                job_model.termination_reason = JobTerminationReason.TERMINATED_BY_SERVER
+                job_model.termination_reason = JobTerminationReason.VOLUME_ERROR
                 job_model.termination_reason_message = "Failed to attach volume"
             except Exception:
                 logger.exception(
@@ -686,8 +664,7 @@ async def _attach_volumes(
                     fmt(job_model),
                 )
                 job_model.status = JobStatus.TERMINATING
-                # TODO: Replace with JobTerminationReason.VOLUME_ERROR in 0.19
-                job_model.termination_reason = JobTerminationReason.TERMINATED_BY_SERVER
+                job_model.termination_reason = JobTerminationReason.VOLUME_ERROR
                 job_model.termination_reason_message = "Failed to attach volume"
             finally:
                 job_model.job_runtime_data = job_runtime_data.json()
@@ -700,13 +677,15 @@ async def _attach_volume(
     instance: InstanceModel,
     instance_id: str,
 ):
+    compute = backend.compute()
+    assert isinstance(compute, ComputeWithVolumeSupport)
     volume = volume_model_to_volume(volume_model)
     # Refresh only to check if the volume wasn't deleted before the lock
     await session.refresh(volume_model)
     if volume_model.deleted:
         raise ServerClientError("Cannot attach a deleted volume")
     attachment_data = await common_utils.run_async(
-        backend.compute().attach_volume,
+        compute.attach_volume,
         volume=volume,
         instance_id=instance_id,
     )

dstack/_internal/server/background/tasks/process_volumes.py

@@ -2,6 +2,7 @@ from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import joinedload
 
+from dstack._internal.core.backends.base.compute import ComputeWithVolumeSupport
 from dstack._internal.core.errors import BackendError, BackendNotAvailable
 from dstack._internal.core.models.volumes import VolumeStatus
 from dstack._internal.server.db import get_session_ctx
@@ -81,17 +82,19 @@ async def _process_submitted_volume(session: AsyncSession, volume_model: VolumeM
         await session.commit()
         return
 
+    compute = backend.compute()
+    assert isinstance(compute, ComputeWithVolumeSupport)
     try:
         if volume.configuration.volume_id is not None:
             logger.info("Registering external volume %s", volume_model.name)
             vpd = await run_async(
-                backend.compute().register_volume,
+                compute.register_volume,
                 volume=volume,
             )
         else:
             logger.info("Provisioning new volume %s", volume_model.name)
             vpd = await run_async(
-                backend.compute().create_volume,
+                compute.create_volume,
                 volume=volume,
             )
     except BackendError as e:

dstack/_internal/server/migrations/versions/7bc2586e8b9e_make_instancemodel_pool_id_optional.py

@@ -0,0 +1,36 @@
+"""Make InstanceModel.pool_id optional
+
+Revision ID: 7bc2586e8b9e
+Revises: bc8ca4a505c6
+Create Date: 2025-03-13 11:13:39.748303
+
+"""
+
+import sqlalchemy_utils
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "7bc2586e8b9e"
+down_revision = "bc8ca4a505c6"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("instances", schema=None) as batch_op:
+        batch_op.alter_column(
+            "pool_id", existing_type=sqlalchemy_utils.UUIDType(binary=False), nullable=True
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("instances", schema=None) as batch_op:
+        batch_op.alter_column(
+            "pool_id", existing_type=sqlalchemy_utils.UUIDType(binary=False), nullable=False
+        )
+
+    # ### end Alembic commands ###

dstack/_internal/server/migrations/versions/bc8ca4a505c6_store_backendtype_as_string.py

@@ -0,0 +1,171 @@
+"""Store BackendType as string
+
+Revision ID: bc8ca4a505c6
+Revises: 98d1b92988bc
+Create Date: 2025-03-10 14:49:06.837118
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "bc8ca4a505c6"
+down_revision = "98d1b92988bc"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("backends", schema=None) as batch_op:
+        batch_op.alter_column(
+            "type",
+            existing_type=postgresql.ENUM(
+                "AWS",
+                "AZURE",
+                "CUDO",
+                "DATACRUNCH",
+                "DSTACK",
+                "GCP",
+                "KUBERNETES",
+                "LAMBDA",
+                "LOCAL",
+                "REMOTE",
+                "NEBIUS",
+                "OCI",
+                "RUNPOD",
+                "TENSORDOCK",
+                "VASTAI",
+                "VULTR",
+                name="backendtype",
+            ),
+            type_=sa.String(length=100),
+            existing_nullable=False,
+        )
+
+    with op.batch_alter_table("instances", schema=None) as batch_op:
+        batch_op.alter_column(
+            "backend",
+            existing_type=postgresql.ENUM(
+                "AWS",
+                "AZURE",
+                "CUDO",
+                "DATACRUNCH",
+                "DSTACK",
+                "GCP",
+                "KUBERNETES",
+                "LAMBDA",
+                "LOCAL",
+                "REMOTE",
+                "NEBIUS",
+                "OCI",
+                "RUNPOD",
+                "TENSORDOCK",
+                "VASTAI",
+                "VULTR",
+                name="backendtype",
+            ),
+            type_=sa.String(length=100),
+            existing_nullable=True,
+        )
+
+    sa.Enum(
+        "AWS",
+        "AZURE",
+        "CUDO",
+        "DATACRUNCH",
+        "DSTACK",
+        "GCP",
+        "KUBERNETES",
+        "LAMBDA",
+        "LOCAL",
+        "REMOTE",
+        "NEBIUS",
+        "OCI",
+        "RUNPOD",
+        "TENSORDOCK",
+        "VASTAI",
+        "VULTR",
+        name="backendtype",
+    ).drop(op.get_bind())
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    sa.Enum(
+        "AWS",
+        "AZURE",
+        "CUDO",
+        "DATACRUNCH",
+        "DSTACK",
+        "GCP",
+        "KUBERNETES",
+        "LAMBDA",
+        "LOCAL",
+        "REMOTE",
+        "NEBIUS",
+        "OCI",
+        "RUNPOD",
+        "TENSORDOCK",
+        "VASTAI",
+        "VULTR",
+        name="backendtype",
+    ).create(op.get_bind())
+    with op.batch_alter_table("instances", schema=None) as batch_op:
+        batch_op.alter_column(
+            "backend",
+            existing_type=sa.String(length=100),
+            type_=postgresql.ENUM(
+                "AWS",
+                "AZURE",
+                "CUDO",
+                "DATACRUNCH",
+                "DSTACK",
+                "GCP",
+                "KUBERNETES",
+                "LAMBDA",
+                "LOCAL",
+                "REMOTE",
+                "NEBIUS",
+                "OCI",
+                "RUNPOD",
+                "TENSORDOCK",
+                "VASTAI",
+                "VULTR",
+                name="backendtype",
+            ),
+            existing_nullable=True,
+            postgresql_using="backend::VARCHAR::backendtype",
+        )
+
+    with op.batch_alter_table("backends", schema=None) as batch_op:
+        batch_op.alter_column(
+            "type",
+            existing_type=sa.String(length=100),
+            type_=postgresql.ENUM(
+                "AWS",
+                "AZURE",
+                "CUDO",
+                "DATACRUNCH",
+                "DSTACK",
+                "GCP",
+                "KUBERNETES",
+                "LAMBDA",
+                "LOCAL",
+                "REMOTE",
+                "NEBIUS",
+                "OCI",
+                "RUNPOD",
+                "TENSORDOCK",
+                "VASTAI",
+                "VULTR",
+                name="backendtype",
+            ),
+            existing_nullable=False,
+            postgresql_using="type::VARCHAR::backendtype",
+        )
+
+    # ### end Alembic commands ###