dstack 0.0.9__py3-none-any.whl → 0.20.7__py3-none-any.whl

dstack/_internal/server/background/tasks/process_compute_groups.py

@@ -0,0 +1,167 @@
+import asyncio
+import datetime
+from datetime import timedelta
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import joinedload, load_only
+
+from dstack._internal.core.backends.base.compute import ComputeWithGroupProvisioningSupport
+from dstack._internal.core.errors import BackendError
+from dstack._internal.core.models.compute_groups import ComputeGroupStatus
+from dstack._internal.core.models.instances import InstanceStatus
+from dstack._internal.server.db import get_db, get_session_ctx
+from dstack._internal.server.models import (
+    ComputeGroupModel,
+    ProjectModel,
+)
+from dstack._internal.server.services import backends as backends_services
+from dstack._internal.server.services.compute_groups import compute_group_model_to_compute_group
+from dstack._internal.server.services.instances import switch_instance_status
+from dstack._internal.server.services.locking import get_locker
+from dstack._internal.server.utils import sentry_utils
+from dstack._internal.utils.common import get_current_datetime, run_async
+from dstack._internal.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+MIN_PROCESSING_INTERVAL = timedelta(seconds=30)
+
+TERMINATION_RETRY_TIMEOUT = timedelta(seconds=60)
+TERMINATION_RETRY_MAX_DURATION = timedelta(minutes=15)
+
+
+async def process_compute_groups(batch_size: int = 1):
+    tasks = []
+    for _ in range(batch_size):
+        tasks.append(_process_next_compute_group())
+    await asyncio.gather(*tasks)
+
+
+@sentry_utils.instrument_background_task
+async def _process_next_compute_group():
+    lock, lockset = get_locker(get_db().dialect_name).get_lockset(ComputeGroupModel.__tablename__)
+    async with get_session_ctx() as session:
+        async with lock:
+            res = await session.execute(
+                select(ComputeGroupModel)
+                .where(
+                    ComputeGroupModel.deleted == False,
+                    ComputeGroupModel.id.not_in(lockset),
+                    ComputeGroupModel.last_processed_at
+                    < get_current_datetime() - MIN_PROCESSING_INTERVAL,
+                )
+                .options(load_only(ComputeGroupModel.id))
+                .order_by(ComputeGroupModel.last_processed_at.asc())
+                .limit(1)
+                .with_for_update(skip_locked=True, key_share=True)
+            )
+            compute_group_model = res.scalar()
+            if compute_group_model is None:
+                return
+            compute_group_model_id = compute_group_model.id
+            lockset.add(compute_group_model_id)
+        try:
+            await _process_compute_group(
+                session=session,
+                compute_group_model=compute_group_model,
+            )
+        finally:
+            lockset.difference_update([compute_group_model_id])
+
+
+async def _process_compute_group(session: AsyncSession, compute_group_model: ComputeGroupModel):
+    # Refetch to load related attributes.
+    res = await session.execute(
+        select(ComputeGroupModel)
+        .where(ComputeGroupModel.id == compute_group_model.id)
+        .options(
+            joinedload(ComputeGroupModel.instances),
+            joinedload(ComputeGroupModel.project).joinedload(ProjectModel.backends),
+        )
+        .execution_options(populate_existing=True)
+    )
+    compute_group_model = res.unique().scalar_one()
+    if all(i.status == InstanceStatus.TERMINATING for i in compute_group_model.instances):
+        await _terminate_compute_group(session, compute_group_model)
+    compute_group_model.last_processed_at = get_current_datetime()
+    await session.commit()
+
+
+async def _terminate_compute_group(
+    session: AsyncSession, compute_group_model: ComputeGroupModel
+) -> None:
+    if (
+        compute_group_model.last_termination_retry_at is not None
+        and _next_termination_retry_at(compute_group_model) > get_current_datetime()
+    ):
+        return
+    compute_group = compute_group_model_to_compute_group(compute_group_model)
+    cgpd = compute_group.provisioning_data
+    backend = await backends_services.get_project_backend_by_type(
+        project=compute_group_model.project,
+        backend_type=cgpd.backend,
+    )
+    if backend is None:
+        logger.error(
+            "Failed to terminate compute group %s. Backend %s not available.",
+            compute_group.name,
+            cgpd.backend,
+        )
+    else:
+        logger.debug("Terminating compute group %s", compute_group.name)
+        compute = backend.compute()
+        assert isinstance(compute, ComputeWithGroupProvisioningSupport)
+        try:
+            await run_async(
+                compute.terminate_compute_group,
+                compute_group,
+            )
+        except Exception as e:
+            if compute_group_model.first_termination_retry_at is None:
+                compute_group_model.first_termination_retry_at = get_current_datetime()
+            compute_group_model.last_termination_retry_at = get_current_datetime()
+            if _next_termination_retry_at(compute_group_model) < _get_termination_deadline(
+                compute_group_model
+            ):
+                logger.warning(
+                    "Failed to terminate compute group %s. Will retry. Error: %r",
+                    compute_group.name,
+                    e,
+                    exc_info=not isinstance(e, BackendError),
+                )
+                return
+            logger.error(
+                "Failed all attempts to terminate compute group %s."
+                " Please terminate it manually to avoid unexpected charges."
+                " Error: %r",
+                compute_group.name,
+                e,
+                exc_info=not isinstance(e, BackendError),
+            )
+
+    compute_group_model.deleted = True
+    compute_group_model.deleted_at = get_current_datetime()
+    compute_group_model.status = ComputeGroupStatus.TERMINATED
+    # Terminating instances belonging to a compute group are locked implicitly
+    # by locking the compute group.
+    for instance_model in compute_group_model.instances:
+        instance_model.deleted = True
+        instance_model.deleted_at = get_current_datetime()
+        instance_model.finished_at = get_current_datetime()
+        switch_instance_status(session, instance_model, InstanceStatus.TERMINATED)
+    logger.info(
+        "Terminated compute group %s",
+        compute_group.name,
+    )
+
+
+def _next_termination_retry_at(compute_group_model: ComputeGroupModel) -> datetime.datetime:
+    assert compute_group_model.last_termination_retry_at is not None
+    return compute_group_model.last_termination_retry_at + TERMINATION_RETRY_TIMEOUT
+
+
+def _get_termination_deadline(compute_group_model: ComputeGroupModel) -> datetime.datetime:
+    assert compute_group_model.first_termination_retry_at is not None
+    return compute_group_model.first_termination_retry_at + TERMINATION_RETRY_MAX_DURATION

dstack/_internal/server/background/tasks/process_events.py

@@ -0,0 +1,17 @@
+from datetime import timedelta
+
+from sqlalchemy import delete
+
+from dstack._internal.server import settings
+from dstack._internal.server.db import get_session_ctx
+from dstack._internal.server.models import EventModel
+from dstack._internal.server.utils import sentry_utils
+from dstack._internal.utils.common import get_current_datetime
+
+
+@sentry_utils.instrument_background_task
+async def delete_events():
+    cutoff = get_current_datetime() - timedelta(seconds=settings.SERVER_EVENTS_TTL_SECONDS)
+    stmt = delete(EventModel).where(EventModel.recorded_at < cutoff)
+    async with get_session_ctx() as session:
+        await session.execute(stmt)

dstack/_internal/server/background/tasks/process_fleets.py

@@ -0,0 +1,289 @@
+from collections import defaultdict
+from datetime import timedelta
+from typing import List
+from uuid import UUID
+
+from sqlalchemy import select, update
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import joinedload, load_only, selectinload, with_loader_criteria
+
+from dstack._internal.core.models.fleets import FleetSpec, FleetStatus
+from dstack._internal.core.models.instances import InstanceStatus, InstanceTerminationReason
+from dstack._internal.server.db import get_db, get_session_ctx
+from dstack._internal.server.models import (
+    FleetModel,
+    InstanceModel,
+    JobModel,
+    PlacementGroupModel,
+    RunModel,
+)
+from dstack._internal.server.services import events
+from dstack._internal.server.services.fleets import (
+    create_fleet_instance_model,
+    get_fleet_spec,
+    get_next_instance_num,
+    is_fleet_empty,
+    is_fleet_in_use,
+    switch_fleet_status,
+)
+from dstack._internal.server.services.instances import switch_instance_status
+from dstack._internal.server.services.locking import get_locker
+from dstack._internal.server.utils import sentry_utils
+from dstack._internal.utils.common import get_current_datetime
+from dstack._internal.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+BATCH_SIZE = 10
+MIN_PROCESSING_INTERVAL = timedelta(seconds=30)
+
+
+@sentry_utils.instrument_background_task
+async def process_fleets():
+    fleet_lock, fleet_lockset = get_locker(get_db().dialect_name).get_lockset(
+        FleetModel.__tablename__
+    )
+    instance_lock, instance_lockset = get_locker(get_db().dialect_name).get_lockset(
+        InstanceModel.__tablename__
+    )
+    async with get_session_ctx() as session:
+        async with fleet_lock, instance_lock:
+            res = await session.execute(
+                select(FleetModel)
+                .where(
+                    FleetModel.deleted == False,
+                    FleetModel.id.not_in(fleet_lockset),
+                    FleetModel.last_processed_at
+                    < get_current_datetime() - MIN_PROCESSING_INTERVAL,
+                )
+                .options(
+                    load_only(FleetModel.id, FleetModel.name),
+                    selectinload(FleetModel.instances).load_only(InstanceModel.id),
+                    with_loader_criteria(
+                        InstanceModel, InstanceModel.deleted == False, include_aliases=True
+                    ),
+                )
+                .order_by(FleetModel.last_processed_at.asc())
+                .limit(BATCH_SIZE)
+                .with_for_update(skip_locked=True, key_share=True)
+            )
+            fleet_models = list(res.scalars().unique().all())
+            fleet_ids = [fm.id for fm in fleet_models]
+            res = await session.execute(
+                select(InstanceModel)
+                .where(
+                    InstanceModel.id.not_in(instance_lockset),
+                    InstanceModel.fleet_id.in_(fleet_ids),
+                    InstanceModel.deleted == False,
+                )
+                .options(load_only(InstanceModel.id, InstanceModel.fleet_id))
+                .order_by(InstanceModel.id)
+                .with_for_update(skip_locked=True, key_share=True)
+            )
+            instance_models = list(res.scalars().all())
+            fleet_id_to_locked_instances = defaultdict(list)
+            for instance_model in instance_models:
+                fleet_id_to_locked_instances[instance_model.fleet_id].append(instance_model)
+            # Process only fleets with all instances locked.
+            # Other fleets won't be processed but will still be locked to avoid new transaction.
+            # This should not be problematic as long as process_fleets is quick.
+            fleet_models_to_process = []
+            for fleet_model in fleet_models:
+                if len(fleet_model.instances) == len(fleet_id_to_locked_instances[fleet_model.id]):
+                    fleet_models_to_process.append(fleet_model)
+                else:
+                    logger.debug(
+                        "Fleet %s processing will be skipped: some instance were not locked",
+                        fleet_model.name,
+                    )
+            for fleet_id in fleet_ids:
+                fleet_lockset.add(fleet_id)
+            instance_ids = [im.id for im in instance_models]
+            for instance_id in instance_ids:
+                instance_lockset.add(instance_id)
+        try:
+            await _process_fleets(session=session, fleet_models=fleet_models_to_process)
+        finally:
+            fleet_lockset.difference_update(fleet_ids)
+            instance_lockset.difference_update(instance_ids)
+
+
+async def _process_fleets(session: AsyncSession, fleet_models: List[FleetModel]):
+    fleet_ids = [fm.id for fm in fleet_models]
+    # Refetch to load related attributes.
+    res = await session.execute(
+        select(FleetModel)
+        .where(FleetModel.id.in_(fleet_ids))
+        .options(
+            joinedload(FleetModel.instances).joinedload(InstanceModel.jobs).load_only(JobModel.id),
+            with_loader_criteria(
+                InstanceModel, InstanceModel.deleted == False, include_aliases=True
+            ),
+        )
+        .options(joinedload(FleetModel.project))
+        .options(joinedload(FleetModel.runs).load_only(RunModel.status))
+        .execution_options(populate_existing=True)
+    )
+    fleet_models = list(res.unique().scalars().all())
+
+    # TODO: Drop fleets auto-deletion after dropping fleets auto-creation.
+    deleted_fleets_ids = []
+    for fleet_model in fleet_models:
+        _consolidate_fleet_state_with_spec(session, fleet_model)
+        deleted = _autodelete_fleet(session, fleet_model)
+        if deleted:
+            deleted_fleets_ids.append(fleet_model.id)
+        fleet_model.last_processed_at = get_current_datetime()
+    await _update_deleted_fleets_placement_groups(session, deleted_fleets_ids)
+    await session.commit()
+
+
+def _consolidate_fleet_state_with_spec(session: AsyncSession, fleet_model: FleetModel):
+    if fleet_model.status == FleetStatus.TERMINATING:
+        return
+    fleet_spec = get_fleet_spec(fleet_model)
+    if fleet_spec.configuration.nodes is None or fleet_spec.autocreated:
+        # Only explicitly created cloud fleets are consolidated.
+        return
+    if not _is_fleet_ready_for_consolidation(fleet_model):
+        return
+    changed_instances = _maintain_fleet_nodes_in_min_max_range(session, fleet_model, fleet_spec)
+    if changed_instances:
+        fleet_model.consolidation_attempt += 1
+    else:
+        # The fleet is already consolidated or consolidation is in progress.
+        # We reset consolidation_attempt in both cases for simplicity.
+        # The second case does not need reset but is ok to do since
+        # it means consolidation is longer than delay, so it won't happen too often.
+        # TODO: Reset consolidation_attempt on fleet in-place update.
+        fleet_model.consolidation_attempt = 0
+    fleet_model.last_consolidated_at = get_current_datetime()
+
+
+def _is_fleet_ready_for_consolidation(fleet_model: FleetModel) -> bool:
+    consolidation_retry_delay = _get_consolidation_retry_delay(fleet_model.consolidation_attempt)
+    last_consolidated_at = fleet_model.last_consolidated_at or fleet_model.last_processed_at
+    duration_since_last_consolidation = get_current_datetime() - last_consolidated_at
+    return duration_since_last_consolidation >= consolidation_retry_delay
+
+
+# We use exponentially increasing consolidation retry delays so that
+# consolidation does not happen too often. In particular, this prevents
+# retrying instance provisioning constantly in case of no offers.
+# TODO: Adjust delays.
+_CONSOLIDATION_RETRY_DELAYS = [
+    timedelta(seconds=30),
+    timedelta(minutes=1),
+    timedelta(minutes=2),
+    timedelta(minutes=5),
+    timedelta(minutes=10),
+]
+
+
+def _get_consolidation_retry_delay(consolidation_attempt: int) -> timedelta:
+    if consolidation_attempt < len(_CONSOLIDATION_RETRY_DELAYS):
+        return _CONSOLIDATION_RETRY_DELAYS[consolidation_attempt]
+    return _CONSOLIDATION_RETRY_DELAYS[-1]
+
+
+def _maintain_fleet_nodes_in_min_max_range(
+    session: AsyncSession,
+    fleet_model: FleetModel,
+    fleet_spec: FleetSpec,
+) -> bool:
+    """
+    Ensures the fleet has at least `nodes.min` and at most `nodes.max` instances.
+    Returns `True` if retried, added new instances, or terminated redundant instances and `False` otherwise.
+    """
+    assert fleet_spec.configuration.nodes is not None
+    for instance in fleet_model.instances:
+        # Delete terminated but not deleted instances since
+        # they are going to be replaced with new pending instances.
+        if instance.status == InstanceStatus.TERMINATED and not instance.deleted:
+            instance.deleted = True
+            instance.deleted_at = get_current_datetime()
+    active_instances = [i for i in fleet_model.instances if not i.deleted]
+    active_instances_num = len(active_instances)
+    if active_instances_num >= fleet_spec.configuration.nodes.min:
+        if (
+            fleet_spec.configuration.nodes.max is None
+            or active_instances_num <= fleet_spec.configuration.nodes.max
+        ):
+            return False
+        # Fleet has more instances than allowed by nodes.max.
+        # This is possible due to race conditions (e.g. provisioning jobs in a fleet concurrently)
+        # or if nodes.max is updated.
+        nodes_redundant = active_instances_num - fleet_spec.configuration.nodes.max
+        for instance in fleet_model.instances:
+            if nodes_redundant == 0:
+                break
+            if instance.status in [InstanceStatus.IDLE]:
+                instance.termination_reason = InstanceTerminationReason.MAX_INSTANCES_LIMIT
+                instance.termination_reason_message = "Fleet has too many instances"
+                switch_instance_status(session, instance, InstanceStatus.TERMINATING)
+                nodes_redundant -= 1
+        return True
+    nodes_missing = fleet_spec.configuration.nodes.min - active_instances_num
+    for i in range(nodes_missing):
+        instance_model = create_fleet_instance_model(
+            session=session,
+            project=fleet_model.project,
+            # TODO: Store fleet.user and pass it instead of the project owner.
+            username=fleet_model.project.owner.name,
+            spec=fleet_spec,
+            instance_num=get_next_instance_num({i.instance_num for i in active_instances}),
+        )
+        events.emit(
+            session,
+            (
+                "Instance created to meet target fleet node count."
+                f" Status: {instance_model.status.upper()}"
+            ),
+            actor=events.SystemActor(),
+            targets=[events.Target.from_model(instance_model)],
+        )
+        active_instances.append(instance_model)
+        fleet_model.instances.append(instance_model)
+    logger.info("Added %s instances to fleet %s", nodes_missing, fleet_model.name)
+    return True
+
+
+def _autodelete_fleet(session: AsyncSession, fleet_model: FleetModel) -> bool:
+    if fleet_model.project.deleted:
+        # It used to be possible to delete project with active resources:
+        # https://github.com/dstackai/dstack/issues/3077
+        switch_fleet_status(session, fleet_model, FleetStatus.TERMINATED)
+        fleet_model.deleted = True
+        logger.info("Fleet %s deleted due to deleted project", fleet_model.name)
+        return True
+
+    if is_fleet_in_use(fleet_model) or not is_fleet_empty(fleet_model):
+        return False
+
+    fleet_spec = get_fleet_spec(fleet_model)
+    if (
+        fleet_model.status != FleetStatus.TERMINATING
+        and fleet_spec.configuration.nodes is not None
+        and fleet_spec.configuration.nodes.min == 0
+    ):
+        # Empty fleets that allow 0 nodes should not be auto-deleted
+        return False
+
+    logger.info("Automatic cleanup of an empty fleet %s", fleet_model.name)
+    switch_fleet_status(session, fleet_model, FleetStatus.TERMINATED)
+    fleet_model.deleted = True
+    logger.info("Fleet %s deleted", fleet_model.name)
+    return True
+
+
+async def _update_deleted_fleets_placement_groups(session: AsyncSession, fleets_ids: list[UUID]):
+    if len(fleets_ids) == 0:
+        return
+    await session.execute(
+        update(PlacementGroupModel)
+        .where(
+            PlacementGroupModel.fleet_id.in_(fleets_ids),
+        )
+        .values(fleet_deleted=True)
+    )

dstack/_internal/server/background/tasks/process_gateways.py

@@ -0,0 +1,188 @@
+import asyncio
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import joinedload, lazyload
+
+from dstack._internal.core.errors import BackendError, BackendNotAvailable, SSHError
+from dstack._internal.core.models.gateways import GatewayStatus
+from dstack._internal.server.db import get_db, get_session_ctx
+from dstack._internal.server.models import GatewayComputeModel, GatewayModel, ProjectModel
+from dstack._internal.server.services import backends as backends_services
+from dstack._internal.server.services import gateways as gateways_services
+from dstack._internal.server.services.gateways import (
+    GatewayConnection,
+    create_gateway_compute,
+    gateway_connections_pool,
+    switch_gateway_status,
+)
+from dstack._internal.server.services.locking import advisory_lock_ctx, get_locker
+from dstack._internal.server.services.logging import fmt
+from dstack._internal.server.utils import sentry_utils
+from dstack._internal.utils.common import get_current_datetime
+from dstack._internal.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+async def process_gateways_connections():
+    await _remove_inactive_connections()
+    await _process_active_connections()
+
+
+@sentry_utils.instrument_background_task
+async def process_gateways():
+    lock, lockset = get_locker(get_db().dialect_name).get_lockset(GatewayModel.__tablename__)
+    async with get_session_ctx() as session:
+        async with lock:
+            res = await session.execute(
+                select(GatewayModel)
+                .where(
+                    GatewayModel.status.in_([GatewayStatus.SUBMITTED, GatewayStatus.PROVISIONING]),
+                    GatewayModel.id.not_in(lockset),
+                )
+                .options(lazyload(GatewayModel.gateway_compute))
+                .order_by(GatewayModel.last_processed_at.asc())
+                .limit(1)
+                .with_for_update(skip_locked=True, key_share=True)
+            )
+            gateway_model = res.scalar()
+            if gateway_model is None:
+                return
+            lockset.add(gateway_model.id)
+        gateway_model_id = gateway_model.id
+        try:
+            initial_status = gateway_model.status
+            if initial_status == GatewayStatus.SUBMITTED:
+                await _process_submitted_gateway(session=session, gateway_model=gateway_model)
+            elif initial_status == GatewayStatus.PROVISIONING:
+                await _process_provisioning_gateway(session=session, gateway_model=gateway_model)
+            else:
+                logger.error(
+                    "%s: unexpected gateway status %r", fmt(gateway_model), initial_status.upper()
+                )
+            gateway_model.last_processed_at = get_current_datetime()
+            await session.commit()
+        finally:
+            lockset.difference_update([gateway_model_id])
+
+
+async def _remove_inactive_connections():
+    async with get_session_ctx() as session:
+        res = await session.execute(
+            select(GatewayComputeModel.ip_address).where(GatewayComputeModel.active == True)
+        )
+    active_connection_ips = set(res.scalars().all())
+    for conn in await gateway_connections_pool.all():
+        if conn.ip_address not in active_connection_ips:
+            await gateway_connections_pool.remove(conn.ip_address)
+
+
+async def _process_active_connections():
+    connections = await gateway_connections_pool.all()
+    # Two server processes on a single host cannot process
+    # gateway connections and init gateway connections concurrently:
+    # Race conditions cause conflicting tunnels being opened.
+    async with get_session_ctx() as session:
+        async with advisory_lock_ctx(
+            bind=session,
+            dialect_name=get_db().dialect_name,
+            resource="gateway_tunnels",
+        ):
+            await asyncio.gather(*(_process_connection(conn) for conn in connections))
+
+
+async def _process_connection(conn: GatewayConnection):
+    try:
+        await conn.check_or_restart()
+    except SSHError as e:
+        logger.error("Connection to gateway %s failed: %s", conn.ip_address, e)
+        return
+
+    await conn.try_collect_stats()
+
+
+async def _process_submitted_gateway(session: AsyncSession, gateway_model: GatewayModel):
+    logger.info("%s: started gateway provisioning", fmt(gateway_model))
+    # Refetch to load related attributes.
+    res = await session.execute(
+        select(GatewayModel)
+        .where(GatewayModel.id == gateway_model.id)
+        .options(joinedload(GatewayModel.project).joinedload(ProjectModel.backends))
+        .execution_options(populate_existing=True)
+    )
+    gateway_model = res.unique().scalar_one()
+    configuration = gateways_services.get_gateway_configuration(gateway_model)
+    try:
+        (
+            backend_model,
+            backend,
+        ) = await backends_services.get_project_backend_with_model_by_type_or_error(
+            project=gateway_model.project, backend_type=configuration.backend
+        )
+    except BackendNotAvailable:
+        gateway_model.status_message = "Backend not available"
+        switch_gateway_status(session, gateway_model, GatewayStatus.FAILED)
+        return
+
+    try:
+        gateway_model.gateway_compute = await create_gateway_compute(
+            backend_compute=backend.compute(),
+            project_name=gateway_model.project.name,
+            configuration=configuration,
+            backend_id=backend_model.id,
+        )
+        session.add(gateway_model)
+        switch_gateway_status(session, gateway_model, GatewayStatus.PROVISIONING)
+    except BackendError as e:
+        status_message = f"Backend error: {repr(e)}"
+        if len(e.args) > 0:
+            status_message = str(e.args[0])
+        gateway_model.status_message = status_message
+        switch_gateway_status(session, gateway_model, GatewayStatus.FAILED)
+    except Exception as e:
+        logger.exception("%s: got exception when creating gateway compute", fmt(gateway_model))
+        gateway_model.status_message = f"Unexpected error: {repr(e)}"
+        switch_gateway_status(session, gateway_model, GatewayStatus.FAILED)
+
+
+async def _process_provisioning_gateway(
+    session: AsyncSession, gateway_model: GatewayModel
+) -> None:
+    # Refetch to load related attributes.
+    res = await session.execute(
+        select(GatewayModel)
+        .where(GatewayModel.id == gateway_model.id)
+        .execution_options(populate_existing=True)
+    )
+    gateway_model = res.unique().scalar_one()
+
+    # Provisioning gateways must have compute.
+    assert gateway_model.gateway_compute is not None
+
+    # FIXME: problems caused by blocking on connect_to_gateway_with_retry and configure_gateway:
+    # - cannot delete the gateway before it is provisioned because the DB model is locked
+    # - connection retry counter is reset on server restart
+    # - only one server replica is processing the gateway
+    # Easy to fix by doing only one connection/configuration attempt per processing iteration. The
+    # main challenge is applying the same provisioning model to the dstack Sky gateway to avoid
+    # maintaining a different model for Sky.
+    connection = await gateways_services.connect_to_gateway_with_retry(
+        gateway_model.gateway_compute
+    )
+    if connection is None:
+        gateway_model.status_message = "Failed to connect to gateway"
+        switch_gateway_status(session, gateway_model, GatewayStatus.FAILED)
+        gateway_model.gateway_compute.deleted = True
+        return
+    try:
+        await gateways_services.configure_gateway(connection)
+    except Exception:
+        logger.exception("%s: failed to configure gateway", fmt(gateway_model))
+        gateway_model.status_message = "Failed to configure gateway"
+        switch_gateway_status(session, gateway_model, GatewayStatus.FAILED)
+        await gateway_connections_pool.remove(gateway_model.gateway_compute.ip_address)
+        gateway_model.gateway_compute.active = False
+        return
+
+    switch_gateway_status(session, gateway_model, GatewayStatus.RUNNING)