dstack 0.19.27__py3-none-any.whl → 0.19.28__py3-none-any.whl

dstack/_internal/core/models/resources.py

@@ -7,7 +7,7 @@ from pydantic import Field, parse_obj_as, root_validator, validator
 from pydantic.generics import GenericModel
 from typing_extensions import Annotated
 
-from dstack._internal.core.models.common import CoreModel
+from dstack._internal.core.models.common import CoreConfig, CoreModel, generate_dual_core_model
 from dstack._internal.utils.common import pretty_resources
 from dstack._internal.utils.json_schema import add_extra_schema_types
 from dstack._internal.utils.logging import get_logger
@@ -129,21 +129,22 @@ DEFAULT_MEMORY_SIZE = Range[Memory](min=Memory.parse("8GB"))
 DEFAULT_GPU_COUNT = Range[int](min=1)
 
 
-class CPUSpec(CoreModel):
+class CPUSpecConfig(CoreConfig):
+    @staticmethod
+    def schema_extra(schema: Dict[str, Any]):
+        add_extra_schema_types(
+            schema["properties"]["count"],
+            extra_types=[{"type": "integer"}, {"type": "string"}],
+        )
+
+
+class CPUSpec(generate_dual_core_model(CPUSpecConfig)):
     arch: Annotated[
         Optional[gpuhunt.CPUArchitecture],
         Field(description="The CPU architecture, one of: `x86`, `arm`"),
     ] = None
     count: Annotated[Range[int], Field(description="The number of CPU cores")] = DEFAULT_CPU_COUNT
 
-    class Config(CoreModel.Config):
-        @staticmethod
-        def schema_extra(schema: Dict[str, Any]):
-            add_extra_schema_types(
-                schema["properties"]["count"],
-                extra_types=[{"type": "integer"}, {"type": "string"}],
-            )
-
     @classmethod
     def __get_validators__(cls):
         yield cls.parse
@@ -190,7 +191,28 @@ class CPUSpec(CoreModel):
         return v
 
 
-class GPUSpec(CoreModel):
+class GPUSpecConfig(CoreConfig):
+    @staticmethod
+    def schema_extra(schema: Dict[str, Any]):
+        add_extra_schema_types(
+            schema["properties"]["count"],
+            extra_types=[{"type": "integer"}, {"type": "string"}],
+        )
+        add_extra_schema_types(
+            schema["properties"]["name"],
+            extra_types=[{"type": "string"}],
+        )
+        add_extra_schema_types(
+            schema["properties"]["memory"],
+            extra_types=[{"type": "integer"}, {"type": "string"}],
+        )
+        add_extra_schema_types(
+            schema["properties"]["total_memory"],
+            extra_types=[{"type": "integer"}, {"type": "string"}],
+        )
+
+
+class GPUSpec(generate_dual_core_model(GPUSpecConfig)):
     vendor: Annotated[
         Optional[gpuhunt.AcceleratorVendor],
         Field(
@@ -218,26 +240,6 @@ class GPUSpec(CoreModel):
         Field(description="The minimum compute capability of the GPU (e.g., `7.5`)"),
     ] = None
 
-    class Config(CoreModel.Config):
-        @staticmethod
-        def schema_extra(schema: Dict[str, Any]):
-            add_extra_schema_types(
-                schema["properties"]["count"],
-                extra_types=[{"type": "integer"}, {"type": "string"}],
-            )
-            add_extra_schema_types(
-                schema["properties"]["name"],
-                extra_types=[{"type": "string"}],
-            )
-            add_extra_schema_types(
-                schema["properties"]["memory"],
-                extra_types=[{"type": "integer"}, {"type": "string"}],
-            )
-            add_extra_schema_types(
-                schema["properties"]["total_memory"],
-                extra_types=[{"type": "integer"}, {"type": "string"}],
-            )
-
     @classmethod
     def __get_validators__(cls):
         yield cls.parse
@@ -317,16 +319,17 @@ class GPUSpec(CoreModel):
         return gpuhunt.AcceleratorVendor.cast(v)
 
 
-class DiskSpec(CoreModel):
-    size: Annotated[Range[Memory], Field(description="Disk size")]
+class DiskSpecConfig(CoreConfig):
+    @staticmethod
+    def schema_extra(schema: Dict[str, Any]):
+        add_extra_schema_types(
+            schema["properties"]["size"],
+            extra_types=[{"type": "integer"}, {"type": "string"}],
+        )
 
-    class Config(CoreModel.Config):
-        @staticmethod
-        def schema_extra(schema: Dict[str, Any]):
-            add_extra_schema_types(
-                schema["properties"]["size"],
-                extra_types=[{"type": "integer"}, {"type": "string"}],
-            )
+
+class DiskSpec(generate_dual_core_model(DiskSpecConfig)):
+    size: Annotated[Range[Memory], Field(description="Disk size")]
 
     @classmethod
     def __get_validators__(cls):
@@ -343,7 +346,32 @@ class DiskSpec(CoreModel):
 DEFAULT_DISK = DiskSpec(size=Range[Memory](min=Memory.parse("100GB"), max=None))
 
 
-class ResourcesSpec(CoreModel):
+class ResourcesSpecConfig(CoreConfig):
+    @staticmethod
+    def schema_extra(schema: Dict[str, Any]):
+        add_extra_schema_types(
+            schema["properties"]["cpu"],
+            extra_types=[{"type": "integer"}, {"type": "string"}],
+        )
+        add_extra_schema_types(
+            schema["properties"]["memory"],
+            extra_types=[{"type": "integer"}, {"type": "string"}],
+        )
+        add_extra_schema_types(
+            schema["properties"]["shm_size"],
+            extra_types=[{"type": "integer"}, {"type": "string"}],
+        )
+        add_extra_schema_types(
+            schema["properties"]["gpu"],
+            extra_types=[{"type": "integer"}, {"type": "string"}],
+        )
+        add_extra_schema_types(
+            schema["properties"]["disk"],
+            extra_types=[{"type": "integer"}, {"type": "string"}],
+        )
+
+
+class ResourcesSpec(generate_dual_core_model(ResourcesSpecConfig)):
     # TODO: Remove Range[int] in 0.20. Range[int] for backward compatibility only.
     cpu: Annotated[Union[CPUSpec, Range[int]], Field(description="The CPU requirements")] = (
         CPUSpec()
@@ -362,30 +390,6 @@ class ResourcesSpec(CoreModel):
     gpu: Annotated[Optional[GPUSpec], Field(description="The GPU requirements")] = None
     disk: Annotated[Optional[DiskSpec], Field(description="The disk resources")] = DEFAULT_DISK
 
-    class Config(CoreModel.Config):
-        @staticmethod
-        def schema_extra(schema: Dict[str, Any]):
-            add_extra_schema_types(
-                schema["properties"]["cpu"],
-                extra_types=[{"type": "integer"}, {"type": "string"}],
-            )
-            add_extra_schema_types(
-                schema["properties"]["memory"],
-                extra_types=[{"type": "integer"}, {"type": "string"}],
-            )
-            add_extra_schema_types(
-                schema["properties"]["shm_size"],
-                extra_types=[{"type": "integer"}, {"type": "string"}],
-            )
-            add_extra_schema_types(
-                schema["properties"]["gpu"],
-                extra_types=[{"type": "integer"}, {"type": "string"}],
-            )
-            add_extra_schema_types(
-                schema["properties"]["disk"],
-                extra_types=[{"type": "integer"}, {"type": "string"}],
-            )
-
     def pretty_format(self) -> str:
         # TODO: Remove in 0.20. Use self.cpu directly
         cpu = parse_obj_as(CPUSpec, self.cpu)

dstack/_internal/core/models/runs.py

@@ -1,13 +1,20 @@
 from datetime import datetime, timedelta
 from enum import Enum
-from typing import Any, Dict, List, Literal, Optional, Type
+from typing import Any, Dict, List, Literal, Optional
 from urllib.parse import urlparse
 
 from pydantic import UUID4, Field, root_validator
 from typing_extensions import Annotated
 
 from dstack._internal.core.models.backends.base import BackendType
-from dstack._internal.core.models.common import ApplyAction, CoreModel, NetworkMode, RegistryAuth
+from dstack._internal.core.models.common import (
+    ApplyAction,
+    CoreConfig,
+    CoreModel,
+    NetworkMode,
+    RegistryAuth,
+    generate_dual_core_model,
+)
 from dstack._internal.core.models.configurations import (
     DEFAULT_PROBE_METHOD,
     LEGACY_REPO_DIR,
@@ -385,7 +392,14 @@ class Job(CoreModel):
     job_submissions: List[JobSubmission]
 
 
-class RunSpec(CoreModel):
+class RunSpecConfig(CoreConfig):
+    @staticmethod
+    def schema_extra(schema: Dict[str, Any]):
+        prop = schema.get("properties", {})
+        prop.pop("merged_profile", None)
+
+
+class RunSpec(generate_dual_core_model(RunSpecConfig)):
     # TODO: run_name, working_dir are redundant here since they already passed in configuration
     run_name: Annotated[
         Optional[str],
@@ -458,12 +472,6 @@ class RunSpec(CoreModel):
     # TODO: make merged_profile a computed field after migrating to pydanticV2
     merged_profile: Annotated[Profile, Field(exclude=True)] = None
 
-    class Config(CoreModel.Config):
-        @staticmethod
-        def schema_extra(schema: Dict[str, Any], model: Type) -> None:
-            prop = schema.get("properties", {})
-            prop.pop("merged_profile", None)
-
     @root_validator
     def _merged_profile(cls, values) -> Dict:
         if values.get("profile") is None:

dstack/_internal/server/app.py

@@ -160,6 +160,11 @@ async def lifespan(app: FastAPI):
         logger.info("Background processing is disabled")
     PROBES_SCHEDULER.start()
     dstack_version = DSTACK_VERSION if DSTACK_VERSION else "(no version)"
+    logger.info(
+        "Job network mode: %s (%d)",
+        settings.JOB_NETWORK_MODE.name,
+        settings.JOB_NETWORK_MODE.value,
+    )
     logger.info(f"The admin token is {admin.token.get_plaintext_or_error()}", {"show_path": False})
     logger.info(
         f"The dstack server {dstack_version} is running at {SERVER_URL}",

dstack/_internal/server/background/tasks/process_fleets.py

@@ -177,6 +177,14 @@ def _maintain_fleet_nodes_min(
 
 
 def _autodelete_fleet(fleet_model: FleetModel) -> bool:
+    if fleet_model.project.deleted:
+        # It used to be possible to delete project with active resources:
+        # https://github.com/dstackai/dstack/issues/3077
+        fleet_model.status = FleetStatus.TERMINATED
+        fleet_model.deleted = True
+        logger.info("Fleet %s deleted due to deleted project", fleet_model.name)
+        return True
+
     if is_fleet_in_use(fleet_model) or not is_fleet_empty(fleet_model):
         return False
 

dstack/_internal/server/background/tasks/process_submitted_jobs.py

@@ -5,7 +5,7 @@ import uuid
 from datetime import datetime, timedelta
 from typing import List, Optional, Tuple
 
-from sqlalchemy import and_, not_, or_, select
+from sqlalchemy import and_, func, not_, or_, select
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import contains_eager, joinedload, load_only, noload, selectinload
 
@@ -54,6 +54,7 @@ from dstack._internal.server.models import (
 from dstack._internal.server.services.backends import get_project_backend_by_type_or_error
 from dstack._internal.server.services.fleets import (
     fleet_model_to_fleet,
+    generate_fleet_name,
     get_fleet_requirements,
     get_next_instance_num,
 )
@@ -71,7 +72,7 @@ from dstack._internal.server.services.jobs import (
     get_job_configured_volumes,
     get_job_runtime_data,
 )
-from dstack._internal.server.services.locking import get_locker
+from dstack._internal.server.services.locking import get_locker, string_to_lock_id
 from dstack._internal.server.services.logging import fmt
 from dstack._internal.server.services.offers import get_offers_by_requirements
 from dstack._internal.server.services.requirements.combine import (
@@ -87,7 +88,6 @@ from dstack._internal.server.services.volumes import (
 )
 from dstack._internal.server.utils import sentry_utils
 from dstack._internal.utils import common as common_utils
-from dstack._internal.utils import env as env_utils
 from dstack._internal.utils.logging import get_logger
 
 logger = get_logger(__name__)
@@ -188,6 +188,7 @@ async def _process_submitted_job(session: AsyncSession, job_model: JobModel):
     run_spec = run.run_spec
     profile = run_spec.merged_profile
     job = find_job(run.jobs, job_model.replica_num, job_model.job_num)
+    multinode = job.job_spec.jobs_per_replica > 1
 
     # Master job chooses fleet for the run.
     # Due to two-step processing, it's saved to job_model.fleet.
@@ -310,6 +311,7 @@ async def _process_submitted_job(session: AsyncSession, job_model: JobModel):
                 session=session,
                 instances_with_offers=fleet_instances_with_offers,
                 job_model=job_model,
+                multinode=multinode,
             )
             job_model.fleet = fleet_model
             job_model.instance_assigned = True
@@ -363,7 +365,8 @@ async def _process_submitted_job(session: AsyncSession, job_model: JobModel):
         job_model.job_provisioning_data = job_provisioning_data.json()
         job_model.status = JobStatus.PROVISIONING
         if fleet_model is None:
-            fleet_model = _create_fleet_model_for_job(
+            fleet_model = await _create_fleet_model_for_job(
+                session=session,
                 project=project,
                 run=run,
             )
@@ -385,7 +388,7 @@ async def _process_submitted_job(session: AsyncSession, job_model: JobModel):
             offer=offer,
             instance_num=instance_num,
         )
-        job_model.job_runtime_data = _prepare_job_runtime_data(offer).json()
+        job_model.job_runtime_data = _prepare_job_runtime_data(offer, multinode).json()
         # Both this task and process_fleets can add instances to fleets.
         # TODO: Ensure this does not violate nodes.max when it's enforced.
         instance.fleet_id = fleet_model.id
@@ -614,6 +617,7 @@ async def _assign_job_to_fleet_instance(
     session: AsyncSession,
     instances_with_offers: list[tuple[InstanceModel, InstanceOfferWithAvailability]],
     job_model: JobModel,
+    multinode: bool,
 ) -> Optional[InstanceModel]:
     if len(instances_with_offers) == 0:
         return None
@@ -643,7 +647,7 @@ async def _assign_job_to_fleet_instance(
     job_model.instance = instance
     job_model.used_instance_id = instance.id
     job_model.job_provisioning_data = instance.job_provisioning_data
-    job_model.job_runtime_data = _prepare_job_runtime_data(offer).json()
+    job_model.job_runtime_data = _prepare_job_runtime_data(offer, multinode).json()
     return instance
 
 
@@ -752,7 +756,8 @@ def _check_can_create_new_instance_in_fleet(fleet: Fleet) -> bool:
     return True
 
 
-def _create_fleet_model_for_job(
+async def _create_fleet_model_for_job(
+    session: AsyncSession,
     project: ProjectModel,
     run: Run,
 ) -> FleetModel:
@@ -760,9 +765,19 @@ def _create_fleet_model_for_job(
     if run.run_spec.configuration.type == "task" and run.run_spec.configuration.nodes > 1:
         placement = InstanceGroupPlacement.CLUSTER
     nodes = _get_nodes_required_num_for_run(run.run_spec)
+
+    lock_namespace = f"fleet_names_{project.name}"
+    # TODO: Lock fleet names on SQLite.
+    # Needs some refactoring so that the lock is released after commit.
+    if get_db().dialect_name == "postgresql":
+        await session.execute(
+            select(func.pg_advisory_xact_lock(string_to_lock_id(lock_namespace)))
+        )
+    fleet_name = await generate_fleet_name(session=session, project=project)
+
     spec = FleetSpec(
         configuration=FleetConfiguration(
-            name=run.run_spec.run_name,
+            name=fleet_name,
             placement=placement,
             reservation=run.run_spec.configuration.reservation,
             nodes=FleetNodesSpec(
@@ -776,7 +791,7 @@ def _create_fleet_model_for_job(
     )
     fleet_model = FleetModel(
         id=uuid.uuid4(),
-        name=run.run_spec.run_name,
+        name=fleet_name,
         project=project,
         status=FleetStatus.ACTIVE,
         spec=spec.json(),
@@ -839,12 +854,17 @@ def _create_instance_model_for_job(
     return instance
 
 
-def _prepare_job_runtime_data(offer: InstanceOfferWithAvailability) -> JobRuntimeData:
+def _prepare_job_runtime_data(
+    offer: InstanceOfferWithAvailability, multinode: bool
+) -> JobRuntimeData:
     if offer.blocks == offer.total_blocks:
-        if env_utils.get_bool("DSTACK_FORCE_BRIDGE_NETWORK"):
+        if settings.JOB_NETWORK_MODE == settings.JobNetworkMode.FORCED_BRIDGE:
             network_mode = NetworkMode.BRIDGE
-        else:
+        elif settings.JOB_NETWORK_MODE == settings.JobNetworkMode.HOST_WHEN_POSSIBLE:
             network_mode = NetworkMode.HOST
+        else:
+            assert settings.JOB_NETWORK_MODE == settings.JobNetworkMode.HOST_FOR_MULTINODE_ONLY
+            network_mode = NetworkMode.HOST if multinode else NetworkMode.BRIDGE
         return JobRuntimeData(
             network_mode=network_mode,
             offer=offer,

dstack/_internal/server/models.py

@@ -24,7 +24,7 @@ from sqlalchemy_utils import UUIDType
 
 from dstack._internal.core.errors import DstackError
 from dstack._internal.core.models.backends.base import BackendType
-from dstack._internal.core.models.common import CoreModel
+from dstack._internal.core.models.common import CoreConfig, generate_dual_core_model
 from dstack._internal.core.models.fleets import FleetStatus
 from dstack._internal.core.models.gateways import GatewayStatus
 from dstack._internal.core.models.health import HealthStatus
@@ -71,7 +71,11 @@ class NaiveDateTime(TypeDecorator):
         return value.replace(tzinfo=timezone.utc)
 
 
-class DecryptedString(CoreModel):
+class DecryptedStringConfig(CoreConfig):
+    arbitrary_types_allowed = True
+
+
+class DecryptedString(generate_dual_core_model(DecryptedStringConfig)):
     """
     A type for representing plaintext strings encrypted with `EncryptedString`.
     Besides the string, stores information if the decryption was successful.
@@ -84,9 +88,6 @@ class DecryptedString(CoreModel):
     decrypted: bool = True
     exc: Optional[Exception] = None
 
-    class Config(CoreModel.Config):
-        arbitrary_types_allowed = True
-
     def get_plaintext_or_error(self) -> str:
         if self.decrypted and self.plaintext is not None:
             return self.plaintext

dstack/_internal/server/schemas/gateways.py

@@ -3,24 +3,25 @@ from typing import Annotated, Any, Dict, List, Optional
 from pydantic import Field
 
 from dstack._internal.core.models.backends.base import BackendType
-from dstack._internal.core.models.common import CoreModel
+from dstack._internal.core.models.common import CoreConfig, CoreModel, generate_dual_core_model
 from dstack._internal.core.models.gateways import GatewayConfiguration
 
 
-class CreateGatewayRequest(CoreModel):
+class CreateGatewayRequestConfig(CoreConfig):
+    @staticmethod
+    def schema_extra(schema: Dict[str, Any]):
+        del schema["properties"]["name"]
+        del schema["properties"]["backend_type"]
+        del schema["properties"]["region"]
+
+
+class CreateGatewayRequest(generate_dual_core_model(CreateGatewayRequestConfig)):
     configuration: GatewayConfiguration
     # Deprecated and unused. Left for compatibility with 0.18 clients.
     name: Annotated[Optional[str], Field(exclude=True)] = None
     backend_type: Annotated[Optional[BackendType], Field(exclude=True)] = None
     region: Annotated[Optional[str], Field(exclude=True)] = None
 
-    class Config(CoreModel.Config):
-        @staticmethod
-        def schema_extra(schema: Dict[str, Any]) -> None:
-            del schema["properties"]["name"]
-            del schema["properties"]["backend_type"]
-            del schema["properties"]["region"]
-
 
 class GetGatewayRequest(CoreModel):
     name: str

dstack/_internal/server/services/backends/handlers.py

@@ -20,6 +20,8 @@ async def delete_backends_safe(
     error: bool = True,
 ):
     try:
+        # FIXME: The checks are not under lock,
+        # so there can be dangling active resources due to race conditions.
         await _check_active_instances(
             session=session,
             project=project,

dstack/_internal/server/services/docker.py

@@ -9,7 +9,11 @@ from pydantic import Field, ValidationError, validator
 from typing_extensions import Annotated
 
 from dstack._internal.core.errors import DockerRegistryError
-from dstack._internal.core.models.common import CoreModel, RegistryAuth
+from dstack._internal.core.models.common import (
+    CoreModel,
+    FrozenCoreModel,
+    RegistryAuth,
+)
 from dstack._internal.server.utils.common import join_byte_stream_checked
 from dstack._internal.utils.dxf import PatchedDXF
 
@@ -31,15 +35,12 @@ class DXFAuthAdapter:
         )
 
 
-class DockerImage(CoreModel):
+class DockerImage(FrozenCoreModel):
     image: str
-    registry: Optional[str]
+    registry: Optional[str] = None
     repo: str
     tag: str
-    digest: Optional[str]
-
-    class Config(CoreModel.Config):
-        frozen = True
+    digest: Optional[str] = None
 
 
 class ImageConfig(CoreModel):

dstack/_internal/server/services/projects.py

@@ -14,8 +14,16 @@ from dstack._internal.core.backends.dstack.models import (
 from dstack._internal.core.backends.models import BackendInfo
 from dstack._internal.core.errors import ForbiddenError, ResourceExistsError, ServerClientError
 from dstack._internal.core.models.projects import Member, MemberPermissions, Project
+from dstack._internal.core.models.runs import RunStatus
 from dstack._internal.core.models.users import GlobalRole, ProjectRole
-from dstack._internal.server.models import MemberModel, ProjectModel, UserModel
+from dstack._internal.server.models import (
+    FleetModel,
+    MemberModel,
+    ProjectModel,
+    RunModel,
+    UserModel,
+    VolumeModel,
+)
 from dstack._internal.server.schemas.projects import MemberSetting
 from dstack._internal.server.services import users
 from dstack._internal.server.services.backends import (
@@ -178,6 +186,19 @@ async def delete_projects(
                 raise ForbiddenError()
         if all(name in projects_names for name in user_project_names):
             raise ServerClientError("Cannot delete the only project")
+
+    res = await session.execute(
+        select(ProjectModel.id).where(ProjectModel.name.in_(projects_names))
+    )
+    project_ids = res.scalars().all()
+    if len(project_ids) != len(projects_names):
+        raise ServerClientError("Failed to delete non-existent projects")
+
+    for project_id in project_ids:
+        # FIXME: The checks are not under lock,
+        # so there can be dangling active resources due to race conditions.
+        await _check_project_has_active_resources(session=session, project_id=project_id)
+
     timestamp = str(int(get_current_datetime().timestamp()))
     new_project_name = "_deleted_" + timestamp + ProjectModel.name
     await session.execute(
@@ -614,6 +635,36 @@ def _is_project_admin(
     return False
 
 
+async def _check_project_has_active_resources(session: AsyncSession, project_id: uuid.UUID):
+    res = await session.execute(
+        select(RunModel.run_name).where(
+            RunModel.project_id == project_id,
+            RunModel.status.not_in(RunStatus.finished_statuses()),
+        )
+    )
+    run_names = list(res.scalars().all())
+    if len(run_names) > 0:
+        raise ServerClientError(f"Failed to delete project with active runs: {run_names}")
+    res = await session.execute(
+        select(FleetModel.name).where(
+            FleetModel.project_id == project_id,
+            FleetModel.deleted.is_(False),
+        )
+    )
+    fleet_names = list(res.scalars().all())
+    if len(fleet_names) > 0:
+        raise ServerClientError(f"Failed to delete project with active fleets: {fleet_names}")
+    res = await session.execute(
+        select(VolumeModel.name).where(
+            VolumeModel.project_id == project_id,
+            VolumeModel.deleted.is_(False),
+        )
+    )
+    volume_names = list(res.scalars().all())
+    if len(volume_names) > 0:
+        raise ServerClientError(f"Failed to delete project with active volumes: {volume_names}")
+
+
 async def remove_project_members(
     session: AsyncSession,
     user: UserModel,

dstack/_internal/server/settings.py

@@ -4,8 +4,14 @@ Environment variables read by the dstack server. Documented in reference/environ
 
 import os
 import warnings
+from enum import Enum
 from pathlib import Path
 
+from dstack._internal.utils.env import environ
+from dstack._internal.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
 DSTACK_DIR_PATH = Path("~/.dstack/").expanduser()
 
 SERVER_DIR_PATH = Path(os.getenv("DSTACK_SERVER_DIR", DSTACK_DIR_PATH / "server"))
@@ -136,3 +142,43 @@ UPDATE_DEFAULT_PROJECT = os.getenv("DSTACK_UPDATE_DEFAULT_PROJECT") is not None
 DO_NOT_UPDATE_DEFAULT_PROJECT = os.getenv("DSTACK_DO_NOT_UPDATE_DEFAULT_PROJECT") is not None
 SKIP_GATEWAY_UPDATE = os.getenv("DSTACK_SKIP_GATEWAY_UPDATE") is not None
 ENABLE_PROMETHEUS_METRICS = os.getenv("DSTACK_ENABLE_PROMETHEUS_METRICS") is not None
+
+
+class JobNetworkMode(Enum):
+    # "host" for multinode runs only, "bridge" otherwise. Opt-in new defaut
+    HOST_FOR_MULTINODE_ONLY = 1
+    # "bridge" if the job occupies only a part of the instance, "host" otherswise. Current default
+    HOST_WHEN_POSSIBLE = 2
+    # Always "bridge", even for multinode runs. Same as legacy DSTACK_FORCE_BRIDGE_NETWORK=true
+    FORCED_BRIDGE = 3
+
+
+def _get_job_network_mode() -> JobNetworkMode:
+    # Current default
+    mode = JobNetworkMode.HOST_WHEN_POSSIBLE
+    bridge_var = "DSTACK_FORCE_BRIDGE_NETWORK"
+    force_bridge = environ.get_bool(bridge_var)
+    mode_var = "DSTACK_SERVER_JOB_NETWORK_MODE"
+    mode_from_env = environ.get_enum(mode_var, JobNetworkMode, value_type=int)
+    if mode_from_env is not None:
+        if force_bridge is not None:
+            logger.warning(
+                f"{bridge_var} is deprecated since 0.19.27 and ignored when {mode_var} is set"
+            )
+        return mode_from_env
+    if force_bridge is not None:
+        if force_bridge:
+            mode = JobNetworkMode.FORCED_BRIDGE
+            logger.warning(
+                (
+                    f"{bridge_var} is deprecated since 0.19.27."
+                    f" Set {mode_var} to {mode.value} and remove {bridge_var}"
+                )
+            )
+        else:
+            logger.warning(f"{bridge_var} is deprecated since 0.19.27. Remove {bridge_var}")
+    return mode
+
+
+JOB_NETWORK_MODE = _get_job_network_mode()
+del _get_job_network_mode