dstack 0.19.27__py3-none-any.whl → 0.19.29__py3-none-any.whl

dstack/_internal/server/background/tasks/process_submitted_jobs.py

@@ -5,7 +5,7 @@ import uuid
 from datetime import datetime, timedelta
 from typing import List, Optional, Tuple
 
-from sqlalchemy import and_, not_, or_, select
+from sqlalchemy import and_, func, not_, or_, select
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import contains_eager, joinedload, load_only, noload, selectinload
 
@@ -54,6 +54,7 @@ from dstack._internal.server.models import (
 from dstack._internal.server.services.backends import get_project_backend_by_type_or_error
 from dstack._internal.server.services.fleets import (
     fleet_model_to_fleet,
+    generate_fleet_name,
     get_fleet_requirements,
     get_next_instance_num,
 )
@@ -71,7 +72,7 @@ from dstack._internal.server.services.jobs import (
     get_job_configured_volumes,
     get_job_runtime_data,
 )
-from dstack._internal.server.services.locking import get_locker
+from dstack._internal.server.services.locking import get_locker, string_to_lock_id
 from dstack._internal.server.services.logging import fmt
 from dstack._internal.server.services.offers import get_offers_by_requirements
 from dstack._internal.server.services.requirements.combine import (
@@ -87,7 +88,6 @@ from dstack._internal.server.services.volumes import (
 )
 from dstack._internal.server.utils import sentry_utils
 from dstack._internal.utils import common as common_utils
-from dstack._internal.utils import env as env_utils
 from dstack._internal.utils.logging import get_logger
 
 logger = get_logger(__name__)
@@ -188,6 +188,7 @@ async def _process_submitted_job(session: AsyncSession, job_model: JobModel):
     run_spec = run.run_spec
     profile = run_spec.merged_profile
     job = find_job(run.jobs, job_model.replica_num, job_model.job_num)
+    multinode = job.job_spec.jobs_per_replica > 1
 
     # Master job chooses fleet for the run.
     # Due to two-step processing, it's saved to job_model.fleet.
@@ -288,7 +289,8 @@ async def _process_submitted_job(session: AsyncSession, job_model: JobModel):
                     instance_filters=instance_filters,
                 )
             fleet_models = fleet_models_with_instances + fleet_models_without_instances
-            fleet_model, fleet_instances_with_offers = _find_optimal_fleet_with_offers(
+            fleet_model, fleet_instances_with_offers = await _find_optimal_fleet_with_offers(
+                project=project,
                 fleet_models=fleet_models,
                 run_model=run_model,
                 run_spec=run.run_spec,
@@ -310,6 +312,7 @@ async def _process_submitted_job(session: AsyncSession, job_model: JobModel):
                 session=session,
                 instances_with_offers=fleet_instances_with_offers,
                 job_model=job_model,
+                multinode=multinode,
             )
             job_model.fleet = fleet_model
             job_model.instance_assigned = True
@@ -363,7 +366,8 @@ async def _process_submitted_job(session: AsyncSession, job_model: JobModel):
         job_model.job_provisioning_data = job_provisioning_data.json()
         job_model.status = JobStatus.PROVISIONING
         if fleet_model is None:
-            fleet_model = _create_fleet_model_for_job(
+            fleet_model = await _create_fleet_model_for_job(
+                session=session,
                 project=project,
                 run=run,
             )
@@ -385,7 +389,7 @@ async def _process_submitted_job(session: AsyncSession, job_model: JobModel):
             offer=offer,
             instance_num=instance_num,
         )
-        job_model.job_runtime_data = _prepare_job_runtime_data(offer).json()
+        job_model.job_runtime_data = _prepare_job_runtime_data(offer, multinode).json()
         # Both this task and process_fleets can add instances to fleets.
         # TODO: Ensure this does not violate nodes.max when it's enforced.
         instance.fleet_id = fleet_model.id
@@ -489,7 +493,8 @@ async def _refetch_fleet_models_with_instances(
     return fleet_models
 
 
-def _find_optimal_fleet_with_offers(
+async def _find_optimal_fleet_with_offers(
+    project: ProjectModel,
     fleet_models: list[FleetModel],
     run_model: RunModel,
     run_spec: RunSpec,
@@ -499,58 +504,99 @@ def _find_optimal_fleet_with_offers(
 ) -> tuple[Optional[FleetModel], list[tuple[InstanceModel, InstanceOfferWithAvailability]]]:
     if run_model.fleet is not None:
         # Using the fleet that was already chosen by the master job
-        fleet_instances_with_offers = _get_fleet_instances_with_offers(
+        fleet_instances_with_pool_offers = _get_fleet_instances_with_pool_offers(
             fleet_model=run_model.fleet,
             run_spec=run_spec,
             job=job,
             master_job_provisioning_data=master_job_provisioning_data,
             volumes=volumes,
         )
-        return run_model.fleet, fleet_instances_with_offers
+        return run_model.fleet, fleet_instances_with_pool_offers
 
     if len(fleet_models) == 0:
         return None, []
 
     nodes_required_num = _get_nodes_required_num_for_run(run_spec)
-    # The current strategy is to first consider fleets that can accommodate
-    # the run without additional provisioning and choose the one with the cheapest offer.
-    # Fallback to fleet with the cheapest offer among all fleets with offers.
+    # The current strategy is first to consider fleets that can accommodate
+    # the run without additional provisioning and choose the one with the cheapest pool offer.
+    # Then choose a fleet with the cheapest pool offer among all fleets with pool offers.
+    # If there are no fleets with pool offers, choose a fleet with a cheapest backend offer.
+    # Fallback to autocreated fleet if fleets have no pool or backend offers.
+    # TODO: Consider trying all backend offers and then choosing a fleet.
     candidate_fleets_with_offers: list[
         tuple[
             Optional[FleetModel],
             list[tuple[InstanceModel, InstanceOfferWithAvailability]],
             int,
-            tuple[int, float],
+            int,
+            tuple[int, float, float],
         ]
     ] = []
     for candidate_fleet_model in fleet_models:
-        fleet_instances_with_offers = _get_fleet_instances_with_offers(
+        fleet_instances_with_pool_offers = _get_fleet_instances_with_pool_offers(
             fleet_model=candidate_fleet_model,
             run_spec=run_spec,
             job=job,
             master_job_provisioning_data=master_job_provisioning_data,
             volumes=volumes,
         )
-        fleet_available_offers = [
-            o for _, o in fleet_instances_with_offers if o.availability.is_available()
-        ]
-        fleet_has_available_capacity = nodes_required_num <= len(fleet_available_offers)
-        fleet_cheapest_offer = math.inf
-        if len(fleet_available_offers) > 0:
-            fleet_cheapest_offer = fleet_available_offers[0].price
-        fleet_priority = (not fleet_has_available_capacity, fleet_cheapest_offer)
+        fleet_has_available_capacity = nodes_required_num <= len(fleet_instances_with_pool_offers)
+        fleet_cheapest_pool_offer = math.inf
+        if len(fleet_instances_with_pool_offers) > 0:
+            fleet_cheapest_pool_offer = fleet_instances_with_pool_offers[0][1].price
+
+        candidate_fleet = fleet_model_to_fleet(candidate_fleet_model)
+        profile = combine_fleet_and_run_profiles(
+            candidate_fleet.spec.merged_profile, run_spec.merged_profile
+        )
+        fleet_requirements = get_fleet_requirements(candidate_fleet.spec)
+        requirements = combine_fleet_and_run_requirements(
+            fleet_requirements, job.job_spec.requirements
+        )
+        multinode = (
+            candidate_fleet.spec.configuration.placement == InstanceGroupPlacement.CLUSTER
+            or job.job_spec.jobs_per_replica > 1
+        )
+        fleet_backend_offers = []
+        if (
+            _check_can_create_new_instance_in_fleet(candidate_fleet)
+            and profile is not None
+            and requirements is not None
+        ):
+            fleet_backend_offers = await get_offers_by_requirements(
+                project=project,
+                profile=profile,
+                requirements=requirements,
+                exclude_not_available=True,
+                multinode=multinode,
+                master_job_provisioning_data=master_job_provisioning_data,
+                volumes=volumes,
+                privileged=job.job_spec.privileged,
+                instance_mounts=check_run_spec_requires_instance_mounts(run_spec),
+            )
+
+        fleet_cheapest_backend_offer = math.inf
+        if len(fleet_backend_offers) > 0:
+            fleet_cheapest_backend_offer = fleet_backend_offers[0][1].price
+
+        fleet_priority = (
+            not fleet_has_available_capacity,
+            fleet_cheapest_pool_offer,
+            fleet_cheapest_backend_offer,
+        )
         candidate_fleets_with_offers.append(
             (
                 candidate_fleet_model,
-                fleet_instances_with_offers,
-                len(fleet_available_offers),
+                fleet_instances_with_pool_offers,
+                len(fleet_instances_with_pool_offers),
+                len(fleet_backend_offers),
                 fleet_priority,
             )
         )
     if run_spec.merged_profile.fleets is None and all(
-        t[2] == 0 for t in candidate_fleets_with_offers
+        t[2] == 0 and t[3] == 0 for t in candidate_fleets_with_offers
     ):
-        # If fleets are not specified and no fleets have available offers, create a new fleet.
+        # If fleets are not specified and no fleets have available pool or backend offers, create a new fleet.
         # This is for compatibility with non-fleet-first UX when runs created new fleets
         # if there are no instances to reuse.
         return None, []
@@ -570,7 +616,7 @@ def _get_nodes_required_num_for_run(run_spec: RunSpec) -> int:
     return nodes_required_num
 
 
-def _get_fleet_instances_with_offers(
+def _get_fleet_instances_with_pool_offers(
     fleet_model: FleetModel,
     run_spec: RunSpec,
     job: Job,
@@ -614,6 +660,7 @@ async def _assign_job_to_fleet_instance(
     session: AsyncSession,
     instances_with_offers: list[tuple[InstanceModel, InstanceOfferWithAvailability]],
     job_model: JobModel,
+    multinode: bool,
 ) -> Optional[InstanceModel]:
     if len(instances_with_offers) == 0:
         return None
@@ -643,7 +690,7 @@ async def _assign_job_to_fleet_instance(
     job_model.instance = instance
     job_model.used_instance_id = instance.id
     job_model.job_provisioning_data = instance.job_provisioning_data
-    job_model.job_runtime_data = _prepare_job_runtime_data(offer).json()
+    job_model.job_runtime_data = _prepare_job_runtime_data(offer, multinode).json()
     return instance
 
 
@@ -752,7 +799,8 @@ def _check_can_create_new_instance_in_fleet(fleet: Fleet) -> bool:
     return True
 
 
-def _create_fleet_model_for_job(
+async def _create_fleet_model_for_job(
+    session: AsyncSession,
     project: ProjectModel,
     run: Run,
 ) -> FleetModel:
@@ -760,9 +808,19 @@ def _create_fleet_model_for_job(
     if run.run_spec.configuration.type == "task" and run.run_spec.configuration.nodes > 1:
         placement = InstanceGroupPlacement.CLUSTER
     nodes = _get_nodes_required_num_for_run(run.run_spec)
+
+    lock_namespace = f"fleet_names_{project.name}"
+    # TODO: Lock fleet names on SQLite.
+    # Needs some refactoring so that the lock is released after commit.
+    if get_db().dialect_name == "postgresql":
+        await session.execute(
+            select(func.pg_advisory_xact_lock(string_to_lock_id(lock_namespace)))
+        )
+    fleet_name = await generate_fleet_name(session=session, project=project)
+
     spec = FleetSpec(
         configuration=FleetConfiguration(
-            name=run.run_spec.run_name,
+            name=fleet_name,
             placement=placement,
             reservation=run.run_spec.configuration.reservation,
             nodes=FleetNodesSpec(
@@ -776,7 +834,7 @@ def _create_fleet_model_for_job(
     )
     fleet_model = FleetModel(
         id=uuid.uuid4(),
-        name=run.run_spec.run_name,
+        name=fleet_name,
         project=project,
         status=FleetStatus.ACTIVE,
         spec=spec.json(),
@@ -839,12 +897,17 @@ def _create_instance_model_for_job(
     return instance
 
 
-def _prepare_job_runtime_data(offer: InstanceOfferWithAvailability) -> JobRuntimeData:
+def _prepare_job_runtime_data(
+    offer: InstanceOfferWithAvailability, multinode: bool
+) -> JobRuntimeData:
     if offer.blocks == offer.total_blocks:
-        if env_utils.get_bool("DSTACK_FORCE_BRIDGE_NETWORK"):
+        if settings.JOB_NETWORK_MODE == settings.JobNetworkMode.FORCED_BRIDGE:
             network_mode = NetworkMode.BRIDGE
-        else:
+        elif settings.JOB_NETWORK_MODE == settings.JobNetworkMode.HOST_WHEN_POSSIBLE:
             network_mode = NetworkMode.HOST
+        else:
+            assert settings.JOB_NETWORK_MODE == settings.JobNetworkMode.HOST_FOR_MULTINODE_ONLY
+            network_mode = NetworkMode.HOST if multinode else NetworkMode.BRIDGE
         return JobRuntimeData(
             network_mode=network_mode,
             offer=offer,

dstack/_internal/server/models.py

@@ -24,7 +24,7 @@ from sqlalchemy_utils import UUIDType
 
 from dstack._internal.core.errors import DstackError
 from dstack._internal.core.models.backends.base import BackendType
-from dstack._internal.core.models.common import CoreModel
+from dstack._internal.core.models.common import CoreConfig, generate_dual_core_model
 from dstack._internal.core.models.fleets import FleetStatus
 from dstack._internal.core.models.gateways import GatewayStatus
 from dstack._internal.core.models.health import HealthStatus
@@ -71,7 +71,11 @@ class NaiveDateTime(TypeDecorator):
         return value.replace(tzinfo=timezone.utc)
 
 
-class DecryptedString(CoreModel):
+class DecryptedStringConfig(CoreConfig):
+    arbitrary_types_allowed = True
+
+
+class DecryptedString(generate_dual_core_model(DecryptedStringConfig)):
     """
     A type for representing plaintext strings encrypted with `EncryptedString`.
     Besides the string, stores information if the decryption was successful.
@@ -84,9 +88,6 @@ class DecryptedString(CoreModel):
     decrypted: bool = True
     exc: Optional[Exception] = None
 
-    class Config(CoreModel.Config):
-        arbitrary_types_allowed = True
-
     def get_plaintext_or_error(self) -> str:
         if self.decrypted and self.plaintext is not None:
             return self.plaintext

dstack/_internal/server/schemas/gateways.py

@@ -3,24 +3,25 @@ from typing import Annotated, Any, Dict, List, Optional
 from pydantic import Field
 
 from dstack._internal.core.models.backends.base import BackendType
-from dstack._internal.core.models.common import CoreModel
+from dstack._internal.core.models.common import CoreConfig, CoreModel, generate_dual_core_model
 from dstack._internal.core.models.gateways import GatewayConfiguration
 
 
-class CreateGatewayRequest(CoreModel):
+class CreateGatewayRequestConfig(CoreConfig):
+    @staticmethod
+    def schema_extra(schema: Dict[str, Any]):
+        del schema["properties"]["name"]
+        del schema["properties"]["backend_type"]
+        del schema["properties"]["region"]
+
+
+class CreateGatewayRequest(generate_dual_core_model(CreateGatewayRequestConfig)):
     configuration: GatewayConfiguration
     # Deprecated and unused. Left for compatibility with 0.18 clients.
     name: Annotated[Optional[str], Field(exclude=True)] = None
     backend_type: Annotated[Optional[BackendType], Field(exclude=True)] = None
     region: Annotated[Optional[str], Field(exclude=True)] = None
 
-    class Config(CoreModel.Config):
-        @staticmethod
-        def schema_extra(schema: Dict[str, Any]) -> None:
-            del schema["properties"]["name"]
-            del schema["properties"]["backend_type"]
-            del schema["properties"]["region"]
-
 
 class GetGatewayRequest(CoreModel):
     name: str

dstack/_internal/server/services/backends/__init__.py

@@ -345,7 +345,7 @@ async def get_instance_offers(
     Returns list of instances satisfying minimal resource requirements sorted by price
     """
     logger.info("Requesting instance offers from backends: %s", [b.TYPE.value for b in backends])
-    tasks = [run_async(backend.compute().get_offers_cached, requirements) for backend in backends]
+    tasks = [run_async(backend.compute().get_offers, requirements) for backend in backends]
     offers_by_backend = []
     for backend, result in zip(backends, await asyncio.gather(*tasks, return_exceptions=True)):
         if isinstance(result, BackendError):

dstack/_internal/server/services/backends/handlers.py

@@ -20,6 +20,8 @@ async def delete_backends_safe(
     error: bool = True,
 ):
     try:
+        # FIXME: The checks are not under lock,
+        # so there can be dangling active resources due to race conditions.
         await _check_active_instances(
             session=session,
             project=project,

dstack/_internal/server/services/docker.py

@@ -9,7 +9,11 @@ from pydantic import Field, ValidationError, validator
 from typing_extensions import Annotated
 
 from dstack._internal.core.errors import DockerRegistryError
-from dstack._internal.core.models.common import CoreModel, RegistryAuth
+from dstack._internal.core.models.common import (
+    CoreModel,
+    FrozenCoreModel,
+    RegistryAuth,
+)
 from dstack._internal.server.utils.common import join_byte_stream_checked
 from dstack._internal.utils.dxf import PatchedDXF
 
@@ -31,15 +35,12 @@ class DXFAuthAdapter:
         )
 
 
-class DockerImage(CoreModel):
+class DockerImage(FrozenCoreModel):
     image: str
-    registry: Optional[str]
+    registry: Optional[str] = None
     repo: str
     tag: str
-    digest: Optional[str]
-
-    class Config(CoreModel.Config):
-        frozen = True
+    digest: Optional[str] = None
 
 
 class ImageConfig(CoreModel):

dstack/_internal/server/services/projects.py

@@ -13,9 +13,22 @@ from dstack._internal.core.backends.dstack.models import (
 )
 from dstack._internal.core.backends.models import BackendInfo
 from dstack._internal.core.errors import ForbiddenError, ResourceExistsError, ServerClientError
-from dstack._internal.core.models.projects import Member, MemberPermissions, Project
+from dstack._internal.core.models.projects import (
+    Member,
+    MemberPermissions,
+    Project,
+    ProjectHookConfig,
+)
+from dstack._internal.core.models.runs import RunStatus
 from dstack._internal.core.models.users import GlobalRole, ProjectRole
-from dstack._internal.server.models import MemberModel, ProjectModel, UserModel
+from dstack._internal.server.models import (
+    FleetModel,
+    MemberModel,
+    ProjectModel,
+    RunModel,
+    UserModel,
+    VolumeModel,
+)
 from dstack._internal.server.schemas.projects import MemberSetting
 from dstack._internal.server.services import users
 from dstack._internal.server.services.backends import (
@@ -112,6 +125,7 @@ async def create_project(
     user: UserModel,
     project_name: str,
     is_public: bool = False,
+    config: Optional[ProjectHookConfig] = None,
 ) -> Project:
     user_permissions = users.get_user_permissions(user)
     if not user_permissions.can_create_projects:
@@ -139,7 +153,7 @@ async def create_project(
         session=session, project_name=project_name
     )
     for hook in _CREATE_PROJECT_HOOKS:
-        await hook(session, project_model)
+        await hook(session, project_model, config)
     # a hook may change project
     session.expire(project_model)
     project_model = await get_project_model_by_name_or_error(
@@ -178,6 +192,19 @@ async def delete_projects(
                 raise ForbiddenError()
         if all(name in projects_names for name in user_project_names):
             raise ServerClientError("Cannot delete the only project")
+
+    res = await session.execute(
+        select(ProjectModel.id).where(ProjectModel.name.in_(projects_names))
+    )
+    project_ids = res.scalars().all()
+    if len(project_ids) != len(projects_names):
+        raise ServerClientError("Failed to delete non-existent projects")
+
+    for project_id in project_ids:
+        # FIXME: The checks are not under lock,
+        # so there can be dangling active resources due to race conditions.
+        await _check_project_has_active_resources(session=session, project_id=project_id)
+
     timestamp = str(int(get_current_datetime().timestamp()))
     new_project_name = "_deleted_" + timestamp + ProjectModel.name
     await session.execute(
@@ -588,7 +615,9 @@ def get_member_permissions(member_model: MemberModel) -> MemberPermissions:
 _CREATE_PROJECT_HOOKS = []
 
 
-def register_create_project_hook(func: Callable[[AsyncSession, ProjectModel], Awaitable[None]]):
+def register_create_project_hook(
+    func: Callable[[AsyncSession, ProjectModel, Optional[ProjectHookConfig]], Awaitable[None]],
+):
     _CREATE_PROJECT_HOOKS.append(func)
 
 
@@ -614,6 +643,36 @@ def _is_project_admin(
     return False
 
 
+async def _check_project_has_active_resources(session: AsyncSession, project_id: uuid.UUID):
+    res = await session.execute(
+        select(RunModel.run_name).where(
+            RunModel.project_id == project_id,
+            RunModel.status.not_in(RunStatus.finished_statuses()),
+        )
+    )
+    run_names = list(res.scalars().all())
+    if len(run_names) > 0:
+        raise ServerClientError(f"Failed to delete project with active runs: {run_names}")
+    res = await session.execute(
+        select(FleetModel.name).where(
+            FleetModel.project_id == project_id,
+            FleetModel.deleted.is_(False),
+        )
+    )
+    fleet_names = list(res.scalars().all())
+    if len(fleet_names) > 0:
+        raise ServerClientError(f"Failed to delete project with active fleets: {fleet_names}")
+    res = await session.execute(
+        select(VolumeModel.name).where(
+            VolumeModel.project_id == project_id,
+            VolumeModel.deleted.is_(False),
+        )
+    )
+    volume_names = list(res.scalars().all())
+    if len(volume_names) > 0:
+        raise ServerClientError(f"Failed to delete project with active volumes: {volume_names}")
+
+
 async def remove_project_members(
     session: AsyncSession,
     user: UserModel,

dstack/_internal/server/services/runs.py

@@ -1164,6 +1164,8 @@ async def process_terminating_run(session: AsyncSession, run_model: RunModel):
         ):
             run_model.next_triggered_at = _get_next_triggered_at(run.run_spec)
             run_model.status = RunStatus.PENDING
+            # Unassign run from fleet so that the new fleet can be chosen on the next submission
+            run_model.fleet = None
         else:
             run_model.status = run_model.termination_reason.to_status()
 

dstack/_internal/server/settings.py

@@ -4,8 +4,14 @@ Environment variables read by the dstack server. Documented in reference/environ
 
 import os
 import warnings
+from enum import Enum
 from pathlib import Path
 
+from dstack._internal.utils.env import environ
+from dstack._internal.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
 DSTACK_DIR_PATH = Path("~/.dstack/").expanduser()
 
 SERVER_DIR_PATH = Path(os.getenv("DSTACK_SERVER_DIR", DSTACK_DIR_PATH / "server"))
@@ -136,3 +142,43 @@ UPDATE_DEFAULT_PROJECT = os.getenv("DSTACK_UPDATE_DEFAULT_PROJECT") is not None
 DO_NOT_UPDATE_DEFAULT_PROJECT = os.getenv("DSTACK_DO_NOT_UPDATE_DEFAULT_PROJECT") is not None
 SKIP_GATEWAY_UPDATE = os.getenv("DSTACK_SKIP_GATEWAY_UPDATE") is not None
 ENABLE_PROMETHEUS_METRICS = os.getenv("DSTACK_ENABLE_PROMETHEUS_METRICS") is not None
+
+
+class JobNetworkMode(Enum):
+    # "host" for multinode runs only, "bridge" otherwise. Opt-in new defaut
+    HOST_FOR_MULTINODE_ONLY = 1
+    # "bridge" if the job occupies only a part of the instance, "host" otherswise. Current default
+    HOST_WHEN_POSSIBLE = 2
+    # Always "bridge", even for multinode runs. Same as legacy DSTACK_FORCE_BRIDGE_NETWORK=true
+    FORCED_BRIDGE = 3
+
+
+def _get_job_network_mode() -> JobNetworkMode:
+    # Current default
+    mode = JobNetworkMode.HOST_WHEN_POSSIBLE
+    bridge_var = "DSTACK_FORCE_BRIDGE_NETWORK"
+    force_bridge = environ.get_bool(bridge_var)
+    mode_var = "DSTACK_SERVER_JOB_NETWORK_MODE"
+    mode_from_env = environ.get_enum(mode_var, JobNetworkMode, value_type=int)
+    if mode_from_env is not None:
+        if force_bridge is not None:
+            logger.warning(
+                f"{bridge_var} is deprecated since 0.19.27 and ignored when {mode_var} is set"
+            )
+        return mode_from_env
+    if force_bridge is not None:
+        if force_bridge:
+            mode = JobNetworkMode.FORCED_BRIDGE
+            logger.warning(
+                (
+                    f"{bridge_var} is deprecated since 0.19.27."
+                    f" Set {mode_var} to {mode.value} and remove {bridge_var}"
+                )
+            )
+        else:
+            logger.warning(f"{bridge_var} is deprecated since 0.19.27. Remove {bridge_var}")
+    return mode
+
+
+JOB_NETWORK_MODE = _get_job_network_mode()
+del _get_job_network_mode

dstack/_internal/server/statics/index.html

@@ -1,3 +1,3 @@
 <!doctype html><html lang="en"><head><meta charset="utf-8"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><title>dstack</title><meta name="description" content="Get GPUs at the best prices and availability from a wide range of providers. No cloud account of your own is required.
 "/><link rel="preconnect" href="https://fonts.googleapis.com"><link rel="preconnect" href="https://fonts.gstatic.com" crossorigin><link href="https://fonts.googleapis.com/css2?family=Roboto:ital,wght@0,100;0,300;0,400;0,500;0,700;0,900;1,100;1,300;1,400;1,500;1,700;1,900&display=swap" rel="stylesheet"><meta name="og:title" content="dstack"><meta name="og:type" content="article"><meta name="og:image" content="/splash_thumbnail.png"><meta name="og:description" content="Get GPUs at the best prices and availability from a wide range of providers. No cloud account of your own is required.
-"><link rel="icon" type="image/x-icon" href="/assets/favicon.ico"><link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png"><link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png"><link rel="icon" type="image/png" sizes="48x48" href="/assets/favicon-48x48.png"><link rel="manifest" href="/assets/manifest.webmanifest"><meta name="mobile-web-app-capable" content="yes"><meta name="theme-color" content="#fff"><meta name="application-name" content="dstackai"><link rel="apple-touch-icon" sizes="57x57" href="/assets/apple-touch-icon-57x57.png"><link rel="apple-touch-icon" sizes="60x60" href="/assets/apple-touch-icon-60x60.png"><link rel="apple-touch-icon" sizes="72x72" href="/assets/apple-touch-icon-72x72.png"><link rel="apple-touch-icon" sizes="76x76" href="/assets/apple-touch-icon-76x76.png"><link rel="apple-touch-icon" sizes="114x114" href="/assets/apple-touch-icon-114x114.png"><link rel="apple-touch-icon" sizes="120x120" href="/assets/apple-touch-icon-120x120.png"><link rel="apple-touch-icon" sizes="144x144" href="/assets/apple-touch-icon-144x144.png"><link rel="apple-touch-icon" sizes="152x152" href="/assets/apple-touch-icon-152x152.png"><link rel="apple-touch-icon" sizes="167x167" href="/assets/apple-touch-icon-167x167.png"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon-180x180.png"><link rel="apple-touch-icon" sizes="1024x1024" href="/assets/apple-touch-icon-1024x1024.png"><meta name="apple-mobile-web-app-capable" content="yes"><meta name="apple-mobile-web-app-status-bar-style" content="black-translucent"><meta name="apple-mobile-web-app-title" content="dstackai"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-640x1136.png"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1136x640.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-750x1334.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1334x750.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1125x2436.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2436x1125.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1170x2532.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2532x1170.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1179x2556.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2556x1179.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-828x1792.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1792x828.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2688.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2688x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2208.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2208x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1284x2778.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2778x1284.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1290x2796.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2796x1290.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1488x2266.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2266x1488.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1536x2048.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2048x1536.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1620x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1620.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1640x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1640.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2388.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2388x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2224.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2224x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-2048x2732.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2732x2048.png"><meta name="msapplication-TileColor" content="#fff"><meta name="msapplication-TileImage" content="/assets/mstile-144x144.png"><meta name="msapplication-config" content="/assets/browserconfig.xml"><link rel="yandex-tableau-widget" href="/assets/yandex-browser-manifest.json"><script defer="defer" src="/main-4eecc75fbe64067eb1bc.js"></script><link href="/main-56191c63d516fd0041c4.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div class="b-page-header" id="header"></div><div id="root"></div><script async src="https://widget.kapa.ai/kapa-widget.bundle.js" data-website-id="11a9339d-20ce-4ddb-9ba3-1b6e29afe8eb" data-project-name="dstack" data-project-color="rgba(0, 0, 0, 0.87)" data-font-size-lg="0.78rem" data-button-hide="true" data-modal-image="/logo-notext.svg" data-modal-z-index="1100" data-modal-title="Ask me anything" data-project-logo="/assets/images/kapa.svg" data-modal-disclaimer="This is a custom LLM for dstack with access to Documentation, API references and GitHub issues. This feature is experimental - Give it a try!" data-user-analytics-fingerprint-enabled="true"></script></body></html>
+"><link rel="icon" type="image/x-icon" href="/assets/favicon.ico"><link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png"><link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png"><link rel="icon" type="image/png" sizes="48x48" href="/assets/favicon-48x48.png"><link rel="manifest" href="/assets/manifest.webmanifest"><meta name="mobile-web-app-capable" content="yes"><meta name="theme-color" content="#fff"><meta name="application-name" content="dstackai"><link rel="apple-touch-icon" sizes="57x57" href="/assets/apple-touch-icon-57x57.png"><link rel="apple-touch-icon" sizes="60x60" href="/assets/apple-touch-icon-60x60.png"><link rel="apple-touch-icon" sizes="72x72" href="/assets/apple-touch-icon-72x72.png"><link rel="apple-touch-icon" sizes="76x76" href="/assets/apple-touch-icon-76x76.png"><link rel="apple-touch-icon" sizes="114x114" href="/assets/apple-touch-icon-114x114.png"><link rel="apple-touch-icon" sizes="120x120" href="/assets/apple-touch-icon-120x120.png"><link rel="apple-touch-icon" sizes="144x144" href="/assets/apple-touch-icon-144x144.png"><link rel="apple-touch-icon" sizes="152x152" href="/assets/apple-touch-icon-152x152.png"><link rel="apple-touch-icon" sizes="167x167" href="/assets/apple-touch-icon-167x167.png"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon-180x180.png"><link rel="apple-touch-icon" sizes="1024x1024" href="/assets/apple-touch-icon-1024x1024.png"><meta name="apple-mobile-web-app-capable" content="yes"><meta name="apple-mobile-web-app-status-bar-style" content="black-translucent"><meta name="apple-mobile-web-app-title" content="dstackai"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-640x1136.png"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1136x640.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-750x1334.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1334x750.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1125x2436.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2436x1125.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1170x2532.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2532x1170.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1179x2556.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2556x1179.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-828x1792.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1792x828.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2688.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2688x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2208.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2208x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1284x2778.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2778x1284.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1290x2796.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2796x1290.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1488x2266.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2266x1488.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1536x2048.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2048x1536.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1620x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1620.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1640x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1640.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2388.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2388x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2224.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2224x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-2048x2732.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2732x2048.png"><meta name="msapplication-TileColor" content="#fff"><meta name="msapplication-TileImage" content="/assets/mstile-144x144.png"><meta name="msapplication-config" content="/assets/browserconfig.xml"><link rel="yandex-tableau-widget" href="/assets/yandex-browser-manifest.json"><script defer="defer" src="/main-c51afa7f243e24d3e446.js"></script><link href="/main-56191fbfe77f49b251de.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div class="b-page-header" id="header"></div><div id="root"></div><script async src="https://widget.kapa.ai/kapa-widget.bundle.js" data-website-id="11a9339d-20ce-4ddb-9ba3-1b6e29afe8eb" data-project-name="dstack" data-project-color="rgba(0, 0, 0, 0.87)" data-font-size-lg="0.78rem" data-button-hide="true" data-modal-image="/logo-notext.svg" data-modal-z-index="1100" data-modal-title="Ask me anything" data-font-family='metro-web, Metro, -apple-system, "system-ui", "Segoe UI", Roboto' data-project-logo="/assets/images/kapa.svg" data-modal-disclaimer="This is a custom LLM for dstack with access to Documentation, API references and GitHub issues. This feature is experimental - Give it a try!" data-user-analytics-fingerprint-enabled="true"></script></body></html>