dstack 0.0.9__py3-none-any.whl → 0.20.7__py3-none-any.whl

dstack/_internal/server/services/volumes.py

@@ -0,0 +1,496 @@
+import uuid
+from datetime import datetime, timedelta
+from typing import List, Optional
+
+from sqlalchemy import and_, func, or_, select
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import joinedload, selectinload
+
+from dstack._internal.core.backends.base.compute import ComputeWithVolumeSupport
+from dstack._internal.core.backends.features import BACKENDS_WITH_VOLUMES_SUPPORT
+from dstack._internal.core.errors import (
+    BackendNotAvailable,
+    ResourceExistsError,
+    ServerClientError,
+)
+from dstack._internal.core.models.volumes import (
+    Volume,
+    VolumeAttachment,
+    VolumeAttachmentData,
+    VolumeConfiguration,
+    VolumeInstance,
+    VolumeProvisioningData,
+    VolumeSpec,
+    VolumeStatus,
+)
+from dstack._internal.core.services import validate_dstack_resource_name
+from dstack._internal.server.db import get_db, is_db_postgres, is_db_sqlite
+from dstack._internal.server.models import (
+    InstanceModel,
+    ProjectModel,
+    UserModel,
+    VolumeAttachmentModel,
+    VolumeModel,
+)
+from dstack._internal.server.services import backends as backends_services
+from dstack._internal.server.services import events
+from dstack._internal.server.services.instances import get_instance_provisioning_data
+from dstack._internal.server.services.locking import (
+    get_locker,
+    string_to_lock_id,
+)
+from dstack._internal.server.services.plugins import apply_plugin_policies
+from dstack._internal.server.services.projects import list_user_project_models
+from dstack._internal.utils import common, random_names
+from dstack._internal.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+def switch_volume_status(
+    session: AsyncSession,
+    volume_model: VolumeModel,
+    new_status: VolumeStatus,
+    actor: events.AnyActor = events.SystemActor(),
+):
+    old_status = volume_model.status
+    if old_status == new_status:
+        return
+
+    volume_model.status = new_status
+
+    msg = f"Volume status changed {old_status.upper()} -> {new_status.upper()}"
+    if volume_model.status_message is not None:
+        msg += f" ({volume_model.status_message})"
+    events.emit(session, msg, actor=actor, targets=[events.Target.from_model(volume_model)])
+
+
+async def list_volumes(
+    session: AsyncSession,
+    user: UserModel,
+    project_name: Optional[str],
+    only_active: bool,
+    prev_created_at: Optional[datetime],
+    prev_id: Optional[uuid.UUID],
+    limit: int,
+    ascending: bool,
+) -> List[Volume]:
+    projects = await list_user_project_models(
+        session=session,
+        user=user,
+        only_names=True,
+    )
+    if project_name is not None:
+        projects = [p for p in projects if p.name == project_name]
+    volume_models = await list_projects_volume_models(
+        session=session,
+        projects=projects,
+        only_active=only_active,
+        prev_created_at=prev_created_at,
+        prev_id=prev_id,
+        limit=limit,
+        ascending=ascending,
+    )
+    return [volume_model_to_volume(v) for v in volume_models]
+
+
+async def list_projects_volume_models(
+    session: AsyncSession,
+    projects: List[ProjectModel],
+    only_active: bool,
+    prev_created_at: Optional[datetime],
+    prev_id: Optional[uuid.UUID],
+    limit: int,
+    ascending: bool,
+) -> List[VolumeModel]:
+    filters = []
+    filters.append(VolumeModel.project_id.in_(p.id for p in projects))
+    if only_active:
+        filters.append(VolumeModel.deleted == False)
+    if prev_created_at is not None:
+        if ascending:
+            if prev_id is None:
+                filters.append(VolumeModel.created_at > prev_created_at)
+            else:
+                filters.append(
+                    or_(
+                        VolumeModel.created_at > prev_created_at,
+                        and_(VolumeModel.created_at == prev_created_at, VolumeModel.id < prev_id),
+                    )
+                )
+        else:
+            if prev_id is None:
+                filters.append(VolumeModel.created_at < prev_created_at)
+            else:
+                filters.append(
+                    or_(
+                        VolumeModel.created_at < prev_created_at,
+                        and_(VolumeModel.created_at == prev_created_at, VolumeModel.id > prev_id),
+                    )
+                )
+    order_by = (VolumeModel.created_at.desc(), VolumeModel.id)
+    if ascending:
+        order_by = (VolumeModel.created_at.asc(), VolumeModel.id.desc())
+    res = await session.execute(
+        select(VolumeModel)
+        .where(*filters)
+        .order_by(*order_by)
+        .limit(limit)
+        .options(joinedload(VolumeModel.user))
+        .options(
+            joinedload(VolumeModel.attachments)
+            .joinedload(VolumeAttachmentModel.instance)
+            .joinedload(InstanceModel.fleet)
+        )
+    )
+    volume_models = list(res.unique().scalars().all())
+    return volume_models
+
+
+async def list_project_volumes(
+    session: AsyncSession,
+    project: ProjectModel,
+    names: Optional[List[str]] = None,
+) -> List[Volume]:
+    volume_models = await list_project_volume_models(session=session, project=project, names=names)
+    return [volume_model_to_volume(v) for v in volume_models]
+
+
+async def list_project_volume_models(
+    session: AsyncSession,
+    project: ProjectModel,
+    names: Optional[List[str]] = None,
+    include_deleted: bool = False,
+) -> List[VolumeModel]:
+    filters = [
+        VolumeModel.project_id == project.id,
+    ]
+    if names is not None:
+        filters.append(VolumeModel.name.in_(names))
+    if not include_deleted:
+        filters.append(VolumeModel.deleted == False)
+    res = await session.execute(
+        select(VolumeModel)
+        .where(*filters)
+        .options(joinedload(VolumeModel.user))
+        .options(
+            joinedload(VolumeModel.attachments)
+            .joinedload(VolumeAttachmentModel.instance)
+            .joinedload(InstanceModel.fleet)
+        )
+    )
+    return list(res.unique().scalars().all())
+
+
+async def get_volume_by_name(
+    session: AsyncSession, project: ProjectModel, name: str
+) -> Optional[Volume]:
+    volume_model = await get_project_volume_model_by_name(
+        session=session, project=project, name=name
+    )
+    if volume_model is None:
+        return None
+    return volume_model_to_volume(volume_model)
+
+
+async def get_project_volume_model_by_name(
+    session: AsyncSession,
+    project: ProjectModel,
+    name: str,
+    include_deleted: bool = False,
+) -> Optional[VolumeModel]:
+    filters = [
+        VolumeModel.name == name,
+        VolumeModel.project_id == project.id,
+    ]
+    if not include_deleted:
+        filters.append(VolumeModel.deleted == False)
+    res = await session.execute(
+        select(VolumeModel)
+        .where(*filters)
+        .options(joinedload(VolumeModel.user))
+        .options(
+            joinedload(VolumeModel.attachments)
+            .joinedload(VolumeAttachmentModel.instance)
+            .joinedload(InstanceModel.fleet)
+        )
+    )
+    return res.unique().scalar_one_or_none()
+
+
+async def create_volume(
+    session: AsyncSession,
+    project: ProjectModel,
+    user: UserModel,
+    configuration: VolumeConfiguration,
+) -> Volume:
+    spec = await apply_plugin_policies(
+        user=user.name,
+        project=project.name,
+        # Create pseudo spec until the volume API is updated to accept spec
+        spec=VolumeSpec(configuration=configuration),
+    )
+    configuration = spec.configuration
+    _validate_volume_configuration(configuration)
+
+    lock_namespace = f"volume_names_{project.name}"
+    if is_db_sqlite():
+        # Start new transaction to see committed changes after lock
+        await session.commit()
+    elif is_db_postgres():
+        await session.execute(
+            select(func.pg_advisory_xact_lock(string_to_lock_id(lock_namespace)))
+        )
+    lock, _ = get_locker(get_db().dialect_name).get_lockset(lock_namespace)
+    async with lock:
+        if configuration.name is not None:
+            volume_model = await get_project_volume_model_by_name(
+                session=session,
+                project=project,
+                name=configuration.name,
+            )
+            if volume_model is not None:
+                raise ResourceExistsError()
+        else:
+            configuration.name = await generate_volume_name(session=session, project=project)
+
+        volume_model = VolumeModel(
+            id=uuid.uuid4(),
+            name=configuration.name,
+            user_id=user.id,
+            project=project,
+            status=VolumeStatus.SUBMITTED,
+            configuration=configuration.json(),
+            attachments=[],
+        )
+        session.add(volume_model)
+        events.emit(
+            session,
+            message=f"Volume created. Status: {volume_model.status.upper()}",
+            actor=events.UserActor.from_user(user),
+            targets=[events.Target.from_model(volume_model)],
+        )
+        await session.commit()
+        return volume_model_to_volume(volume_model)
+
+
+async def delete_volumes(
+    session: AsyncSession, project: ProjectModel, names: List[str], user: UserModel
+):
+    res = await session.execute(
+        select(VolumeModel).where(
+            VolumeModel.project_id == project.id,
+            VolumeModel.name.in_(names),
+            VolumeModel.deleted == False,
+        )
+    )
+    volume_models = res.scalars().all()
+    volumes_ids = sorted([v.id for v in volume_models])
+    await session.commit()
+    logger.info("Deleting volumes: %s", [v.name for v in volume_models])
+    async with get_locker(get_db().dialect_name).lock_ctx(VolumeModel.__tablename__, volumes_ids):
+        # Refetch after lock
+        res = await session.execute(
+            select(VolumeModel)
+            .where(
+                VolumeModel.project_id == project.id,
+                VolumeModel.name.in_(names),
+                VolumeModel.deleted == False,
+            )
+            .options(selectinload(VolumeModel.user))
+            .options(selectinload(VolumeModel.attachments))
+            .execution_options(populate_existing=True)
+            .order_by(VolumeModel.id)  # take locks in order
+            .with_for_update(key_share=True)
+        )
+        volume_models = res.scalars().unique().all()
+        for volume_model in volume_models:
+            if len(volume_model.attachments) > 0:
+                raise ServerClientError(
+                    f"Failed to delete volume {volume_model.name}. Volume is in use."
+                )
+        for volume_model in volume_models:
+            try:
+                await _delete_volume(session=session, project=project, volume_model=volume_model)
+            except Exception:
+                logger.exception("Error when deleting volume %s", volume_model.name)
+            volume_model.deleted = True
+            volume_model.deleted_at = common.get_current_datetime()
+            events.emit(
+                session,
+                message="Volume deleted",
+                actor=events.UserActor.from_user(user),
+                targets=[events.Target.from_model(volume_model)],
+            )
+        await session.commit()
+
+
+def volume_model_to_volume(volume_model: VolumeModel) -> Volume:
+    configuration = get_volume_configuration(volume_model)
+    vpd = get_volume_provisioning_data(volume_model)
+    vad = get_volume_attachment_data(volume_model)
+    # Initially VolumeProvisionigData lacked backend
+    if vpd is not None and vpd.backend is None:
+        vpd.backend = configuration.backend
+    attachments = []
+    for volume_attachment_model in volume_model.attachments:
+        instance = volume_attachment_model.instance
+        attachments.append(
+            VolumeAttachment(
+                instance=instance_model_to_volume_instance(instance),
+                attachment_data=get_attachment_data(volume_attachment_model),
+            )
+        )
+    deleted_at = None
+    if volume_model.deleted_at is not None:
+        deleted_at = volume_model.deleted_at
+    volume = Volume(
+        name=volume_model.name,
+        project_name=volume_model.project.name,
+        user=volume_model.user.name,
+        configuration=configuration,
+        external=configuration.volume_id is not None,
+        created_at=volume_model.created_at,
+        last_processed_at=volume_model.last_processed_at,
+        status=volume_model.status,
+        status_message=volume_model.status_message,
+        deleted=volume_model.deleted,
+        deleted_at=deleted_at,
+        volume_id=vpd.volume_id if vpd is not None else None,
+        provisioning_data=vpd,
+        attachments=attachments,
+        attachment_data=vad,
+        id=volume_model.id,
+    )
+    volume.cost = _get_volume_cost(volume)
+    return volume
+
+
+def get_volume_configuration(volume_model: VolumeModel) -> VolumeConfiguration:
+    return VolumeConfiguration.__response__.parse_raw(volume_model.configuration)
+
+
+def get_volume_provisioning_data(volume_model: VolumeModel) -> Optional[VolumeProvisioningData]:
+    if volume_model.volume_provisioning_data is None:
+        return None
+    return VolumeProvisioningData.__response__.parse_raw(volume_model.volume_provisioning_data)
+
+
+def get_volume_attachment_data(volume_model: VolumeModel) -> Optional[VolumeAttachmentData]:
+    if volume_model.volume_attachment_data is None:
+        return None
+    return VolumeAttachmentData.__response__.parse_raw(volume_model.volume_attachment_data)
+
+
+def get_attachment_data(
+    volume_attachment_model: VolumeAttachmentModel,
+) -> Optional[VolumeAttachmentData]:
+    if volume_attachment_model.attachment_data is None:
+        return None
+    return VolumeAttachmentData.__response__.parse_raw(volume_attachment_model.attachment_data)
+
+
+def instance_model_to_volume_instance(instance_model: InstanceModel) -> VolumeInstance:
+    instance_id = None
+    jpd = get_instance_provisioning_data(instance_model)
+    if jpd is not None:
+        instance_id = jpd.instance_id
+    return VolumeInstance(
+        name=instance_model.name,
+        fleet_name=instance_model.fleet.name if instance_model.fleet else None,
+        instance_num=instance_model.instance_num,
+        instance_id=instance_id,
+    )
+
+
+async def generate_volume_name(session: AsyncSession, project: ProjectModel) -> str:
+    res = await session.execute(
+        select(VolumeModel.name).where(
+            VolumeModel.project_id == project.id,
+            VolumeModel.deleted == False,
+        )
+    )
+    names = set(res.scalars().all())
+    while True:
+        name = random_names.generate_name()
+        if name not in names:
+            return name
+
+
+def _validate_volume_configuration(configuration: VolumeConfiguration):
+    if configuration.volume_id is None and configuration.size is None:
+        raise ServerClientError("Volume must specify either volume_id or size")
+    backends_services.check_backend_type_available(configuration.backend)
+    if configuration.backend not in BACKENDS_WITH_VOLUMES_SUPPORT:
+        raise ServerClientError(
+            f"Volumes are not supported for {configuration.backend.value} backend."
+            f" Available backends with volumes support: {[b.value for b in BACKENDS_WITH_VOLUMES_SUPPORT]}."
+        )
+    if configuration.name is not None:
+        validate_dstack_resource_name(configuration.name)
+
+    if configuration.volume_id is not None and configuration.auto_cleanup_duration is not None:
+        if (
+            isinstance(configuration.auto_cleanup_duration, int)
+            and configuration.auto_cleanup_duration > 0
+        ) or (
+            isinstance(configuration.auto_cleanup_duration, str)
+            and configuration.auto_cleanup_duration not in ("off", "-1")
+        ):
+            raise ServerClientError(
+                "External volumes (with volume_id) do not support auto_cleanup_duration. "
+                "Auto-cleanup only works for volumes created and managed by dstack."
+            )
+
+
+async def _delete_volume(session: AsyncSession, project: ProjectModel, volume_model: VolumeModel):
+    volume = volume_model_to_volume(volume_model)
+    if volume.external:
+        return
+
+    if volume.provisioning_data is None:
+        # The volume wasn't provisioned so there is nothing to delete
+        return
+    if volume.provisioning_data.backend is None:
+        logger.error(
+            f"Failed to delete volume {volume_model.name}. volume.provisioning_data.backend is None."
+        )
+        return
+
+    try:
+        backend = await backends_services.get_project_backend_by_type_or_error(
+            project=volume_model.project,
+            backend_type=volume.provisioning_data.backend,
+        )
+    except BackendNotAvailable:
+        logger.error(
+            f"Failed to delete volume {volume_model.name}. Backend {volume.configuration.backend} not available."
+        )
+        return
+
+    compute = backend.compute()
+    assert isinstance(compute, ComputeWithVolumeSupport)
+    await common.run_async(
+        compute.delete_volume,
+        volume=volume,
+    )
+
+
+# Clouds charge volumes assuming 30-day months, e.g. https://aws.amazon.com/ebs/pricing/
+_VOLUME_PRICING_PERIOD = timedelta(days=30)
+
+
+def _get_volume_cost(volume: Volume) -> float:
+    if volume.provisioning_data is None or volume.provisioning_data.price is None:
+        return 0.0
+    finished_at = common.get_current_datetime()
+    if volume.deleted_at:
+        finished_at = volume.deleted_at
+    elif not volume.status.is_active():
+        finished_at = volume.last_processed_at
+    volume_age = finished_at - volume.created_at
+    return (
+        volume_age.total_seconds()
+        * volume.provisioning_data.price
+        / _VOLUME_PRICING_PERIOD.total_seconds()
+    )

dstack/_internal/server/settings.py

@@ -0,0 +1,161 @@
+"""
+Environment variables read by the dstack server. Documented in reference/environment-variables.md
+"""
+
+import os
+from enum import Enum
+from pathlib import Path
+
+from dstack._internal.utils.env import environ
+from dstack._internal.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+DSTACK_DIR_PATH = Path("~/.dstack/").expanduser()
+
+SERVER_DIR_PATH = Path(os.getenv("DSTACK_SERVER_DIR", DSTACK_DIR_PATH / "server")).resolve()
+
+SERVER_CONFIG_FILE_PATH = SERVER_DIR_PATH / "config.yml"
+
+SERVER_DATA_DIR_PATH = SERVER_DIR_PATH / "data"
+SERVER_DATA_DIR_PATH.mkdir(parents=True, exist_ok=True)
+DATABASE_URL = os.getenv(
+    "DSTACK_DATABASE_URL", f"sqlite+aiosqlite:///{str(SERVER_DATA_DIR_PATH.absolute())}/sqlite.db"
+)
+
+SERVER_HOST = os.getenv("DSTACK_SERVER_HOST", "localhost")
+SERVER_PORT = int(os.getenv("DSTACK_SERVER_PORT", "8000"))
+SERVER_URL = os.getenv("DSTACK_SERVER_URL", f"http://{SERVER_HOST}:{SERVER_PORT}")
+
+SERVER_ENVIRONMENT = os.getenv("DSTACK_SERVER_ENVIRONMENT", "dev")
+
+ROOT_LOG_LEVEL = os.getenv("DSTACK_SERVER_ROOT_LOG_LEVEL", "ERROR").upper()
+LOG_LEVEL = os.getenv("DSTACK_SERVER_LOG_LEVEL", "WARNING").upper()
+LOG_FORMAT = os.getenv("DSTACK_SERVER_LOG_FORMAT", "rich").lower()
+
+ALEMBIC_MIGRATIONS_LOCATION = os.getenv(
+    "DSTACK_ALEMBIC_MIGRATIONS_LOCATION", "dstack._internal.server:migrations"
+)
+
+# Users may want to increase client pool size to support more concurrent resources
+# if their db supports many connections.
+DB_POOL_SIZE = int(os.getenv("DSTACK_DB_POOL_SIZE", 20))
+DB_MAX_OVERFLOW = int(os.getenv("DSTACK_DB_MAX_OVERFLOW", 20))
+
+# Scale the number of background processing tasks
+# allowing to process more resources on one server replica.
+# Not recommended to change on SQLite.
+# DSTACK_DB_POOL_SIZE and DSTACK_DB_MAX_OVERFLOW
+# must be increased proportionally.
+SERVER_BACKGROUND_PROCESSING_FACTOR = int(
+    os.getenv("DSTACK_SERVER_BACKGROUND_PROCESSING_FACTOR", 1)
+)
+
+SERVER_BACKGROUND_PROCESSING_DISABLED = (
+    os.getenv("DSTACK_SERVER_BACKGROUND_PROCESSING_DISABLED") is not None
+)
+SERVER_BACKGROUND_PROCESSING_ENABLED = not SERVER_BACKGROUND_PROCESSING_DISABLED
+
+SERVER_EXECUTOR_MAX_WORKERS = int(os.getenv("DSTACK_SERVER_EXECUTOR_MAX_WORKERS", 128))
+
+MAX_OFFERS_TRIED = int(os.getenv("DSTACK_SERVER_MAX_OFFERS_TRIED", 25))
+MAX_PROBES_PER_JOB = int(os.getenv("DSTACK_SERVER_MAX_PROBES_PER_JOB", 10))
+MAX_PROBE_TIMEOUT = int(os.getenv("DSTACK_SERVER_MAX_PROBE_TIMEOUT", 60 * 5))
+
+SERVER_CONFIG_DISABLED = os.getenv("DSTACK_SERVER_CONFIG_DISABLED") is not None
+SERVER_CONFIG_ENABLED = not SERVER_CONFIG_DISABLED
+
+# TODO: remove deprecated DSTACK_SERVER_BUCKET and DSTACK_SERVER_BUCKET_REGION env var usage
+SERVER_S3_BUCKET = os.getenv("DSTACK_SERVER_S3_BUCKET", os.getenv("DSTACK_SERVER_BUCKET"))
+SERVER_S3_BUCKET_REGION = os.getenv(
+    "DSTACK_SERVER_S3_BUCKET_REGION", os.getenv("DSTACK_SERVER_BUCKET_REGION")
+)
+
+SERVER_GCS_BUCKET = os.getenv("DSTACK_SERVER_GCS_BUCKET")
+
+SERVER_CLOUDWATCH_LOG_GROUP = os.getenv("DSTACK_SERVER_CLOUDWATCH_LOG_GROUP")
+SERVER_CLOUDWATCH_LOG_REGION = os.getenv("DSTACK_SERVER_CLOUDWATCH_LOG_REGION")
+
+SERVER_GCP_LOGGING_PROJECT = os.getenv("DSTACK_SERVER_GCP_LOGGING_PROJECT")
+
+SERVER_FLUENTBIT_HOST = os.getenv("DSTACK_SERVER_FLUENTBIT_HOST")
+SERVER_FLUENTBIT_PORT = int(os.getenv("DSTACK_SERVER_FLUENTBIT_PORT", "24224"))
+SERVER_FLUENTBIT_PROTOCOL = os.getenv("DSTACK_SERVER_FLUENTBIT_PROTOCOL", "forward")
+SERVER_FLUENTBIT_TAG_PREFIX = os.getenv("DSTACK_SERVER_FLUENTBIT_TAG_PREFIX", "dstack")
+
+SERVER_ELASTICSEARCH_HOST = os.getenv("DSTACK_SERVER_ELASTICSEARCH_HOST")
+SERVER_ELASTICSEARCH_INDEX = os.getenv("DSTACK_SERVER_ELASTICSEARCH_INDEX", "dstack-logs")
+SERVER_ELASTICSEARCH_API_KEY = os.getenv("DSTACK_SERVER_ELASTICSEARCH_API_KEY")
+
+SERVER_METRICS_RUNNING_TTL_SECONDS = environ.get_int(
+    "DSTACK_SERVER_METRICS_RUNNING_TTL_SECONDS", default=3600
+)
+SERVER_METRICS_FINISHED_TTL_SECONDS = environ.get_int(
+    "DSTACK_SERVER_METRICS_FINISHED_TTL_SECONDS", default=7 * 24 * 3600
+)
+SERVER_INSTANCE_HEALTH_TTL_SECONDS = environ.get_int(
+    "DSTACK_SERVER_INSTANCE_HEALTH_TTL_SECONDS", default=7 * 24 * 3600
+)
+SERVER_INSTANCE_HEALTH_MIN_COLLECT_INTERVAL_SECONDS = environ.get_int(
+    "DSTACK_SERVER_INSTANCE_HEALTH_MIN_COLLECT_INTERVAL_SECONDS", default=60
+)
+
+SERVER_EVENTS_TTL_SECONDS = int(
+    # default documented in reference/environment-variables.md, keep in sync
+    os.getenv("DSTACK_SERVER_EVENTS_TTL_SECONDS", 30 * 24 * 3600)
+)
+
+SERVER_KEEP_SHIM_TASKS = os.getenv("DSTACK_SERVER_KEEP_SHIM_TASKS") is not None
+
+DEFAULT_PROJECT_NAME = "main"
+
+SENTRY_DSN = os.getenv("DSTACK_SENTRY_DSN")
+SENTRY_TRACES_SAMPLE_RATE = float(os.getenv("DSTACK_SENTRY_TRACES_SAMPLE_RATE", 0.1))
+SENTRY_TRACES_BACKGROUND_SAMPLE_RATE = float(
+    os.getenv("DSTACK_SENTRY_TRACES_BACKGROUND_SAMPLE_RATE", 0.01)
+)
+SENTRY_PROFILES_SAMPLE_RATE = float(os.getenv("DSTACK_SENTRY_PROFILES_SAMPLE_RATE", 0))
+
+DEFAULT_CREDS_DISABLED = os.getenv("DSTACK_DEFAULT_CREDS_DISABLED") is not None
+DEFAULT_CREDS_ENABLED = not DEFAULT_CREDS_DISABLED
+
+ACME_SERVER = os.getenv("DSTACK_ACME_SERVER")
+ACME_EAB_KID = os.getenv("DSTACK_ACME_EAB_KID")
+ACME_EAB_HMAC_KEY = os.getenv("DSTACK_ACME_EAB_HMAC_KEY")
+DEFAULT_SERVICE_CLIENT_MAX_BODY_SIZE = int(
+    os.getenv("DSTACK_DEFAULT_SERVICE_CLIENT_MAX_BODY_SIZE", 64 * 1024 * 1024)
+)
+
+USER_PROJECT_DEFAULT_QUOTA = int(os.getenv("DSTACK_USER_PROJECT_DEFAULT_QUOTA", 10))
+FORBID_SERVICES_WITHOUT_GATEWAY = os.getenv("DSTACK_FORBID_SERVICES_WITHOUT_GATEWAY") is not None
+
+SERVER_CODE_UPLOAD_LIMIT = int(os.getenv("DSTACK_SERVER_CODE_UPLOAD_LIMIT", 2 * 2**20))
+
+# Development settings
+
+SQL_ECHO_ENABLED = os.getenv("DSTACK_SQL_ECHO_ENABLED") is not None
+
+SERVER_PROFILING_ENABLED = os.getenv("DSTACK_SERVER_PROFILING_ENABLED") is not None
+
+UPDATE_DEFAULT_PROJECT = os.getenv("DSTACK_UPDATE_DEFAULT_PROJECT") is not None
+DO_NOT_UPDATE_DEFAULT_PROJECT = os.getenv("DSTACK_DO_NOT_UPDATE_DEFAULT_PROJECT") is not None
+SKIP_GATEWAY_UPDATE = os.getenv("DSTACK_SKIP_GATEWAY_UPDATE") is not None
+ENABLE_PROMETHEUS_METRICS = os.getenv("DSTACK_ENABLE_PROMETHEUS_METRICS") is not None
+
+
+class JobNetworkMode(Enum):
+    # "host" for multinode runs only, "bridge" otherwise. Opt-in new defaut
+    HOST_FOR_MULTINODE_ONLY = 1
+    # "bridge" if the job occupies only a part of the instance, "host" otherswise. Current default
+    HOST_WHEN_POSSIBLE = 2
+    # Always "bridge", even for multinode runs. Same as legacy DSTACK_FORCE_BRIDGE_NETWORK=true
+    FORCED_BRIDGE = 3
+
+
+DEFAULT_JOB_NETWORK_MODE = JobNetworkMode.HOST_WHEN_POSSIBLE
+JOB_NETWORK_MODE = environ.get_enum(
+    "DSTACK_SERVER_JOB_NETWORK_MODE",
+    JobNetworkMode,
+    value_type=int,
+    default=DEFAULT_JOB_NETWORK_MODE,
+)

dstack/_internal/server/statics/4e0eead8c1a73689ef9d.svg

@@ -0,0 +1 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M5.83701 15L4.58751 13.7155L10.1468 8L4.58751 2.28446L5.83701 1L12.6465 8L5.83701 15Z" fill="black"/></svg>

dstack/_internal/server/statics/54a4f50f74c6b9381530.svg

@@ -0,0 +1,7 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 16">
+<g stroke-width="2" stroke="darkorange" shape-rendering="geometricPrecision">
+<polygon stroke-linejoin="round" fill="none" points="8 1 15 15 1 15 8 1"/>
+<rect x="8" y="12" width="0.01" height="0.01"/>
+<line x1="8" y1="6" x2="8" y2="10"/>
+</g>
+</svg>

dstack/_internal/server/statics/68dd1360a7d2611e0132.svg

@@ -0,0 +1,4 @@
+<svg width="20" height="16" viewBox="0 0 20 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M14.7769 14.7337L8.65192 2.48369C8.32946 1.83877 7.40913 1.83877 7.08667 2.48369L0.961669 14.7337C0.670775 15.3155 1.09383 16 1.74429 16H13.9943C14.6448 16 15.0678 15.3155 14.7769 14.7337ZM3.16007 14.25L7.86929 4.83156L12.5785 14.25H3.16007ZM8.74429 11.625V13.375H6.99429V11.625H8.74429ZM6.99429 10.75V7.25H8.74429V10.75H6.99429Z" fill="#EC7211"/>
+<path d="M11.1991 2.95238C10.8809 2.31467 10.3537 1.80526 9.7055 1.509L11.041 1.06978C11.6883 0.949814 12.337 1.27263 12.6317 1.86141L17.6136 11.8161C18.3527 13.2929 17.5938 15.0804 16.018 15.5745C16.4044 14.4507 16.3231 13.2188 15.7924 12.1555L11.1991 2.95238Z" fill="#EC7211"/>
+</svg>