dstack 0.19.6rc1__py3-none-any.whl → 0.19.7__py3-none-any.whl

dstack/_internal/cli/services/configurators/fleet.py

@@ -20,6 +20,7 @@ from dstack._internal.cli.utils.rich import MultiItemStatus
 from dstack._internal.core.errors import (
     CLIError,
     ConfigurationError,
+    MethodNotAllowedError,
     ResourceNotExistsError,
     ServerClientError,
     URLNotFoundError,
@@ -321,7 +322,7 @@ def _print_plan_header(plan: FleetPlan):
             offer.instance.name,
             resources.pretty_format(),
             "yes" if resources.spot else "no",
-            f"${offer.price:g}",
+            f"${offer.price:3f}".rstrip("0").rstrip("."),
             availability,
             style=None if index == 1 else "secondary",
         )
@@ -367,7 +368,7 @@ def _apply_plan(api: Client, plan: FleetPlan) -> Fleet:
             project_name=api.project,
             plan=plan,
         )
-    except URLNotFoundError:
+    except (URLNotFoundError, MethodNotAllowedError):
         # TODO: Remove in 0.20
         return api.client.fleets.create(
             project_name=api.project,

dstack/_internal/cli/services/configurators/run.py

@@ -52,7 +52,7 @@ from dstack.api.utils import load_profile
 _KNOWN_AMD_GPUS = {gpu.name.lower() for gpu in gpuhunt.KNOWN_AMD_GPUS}
 _KNOWN_NVIDIA_GPUS = {gpu.name.lower() for gpu in gpuhunt.KNOWN_NVIDIA_GPUS}
 _KNOWN_TPU_VERSIONS = {gpu.name.lower() for gpu in gpuhunt.KNOWN_TPUS}
-
+_KNOWN_TENSTORRENT_GPUS = {gpu.name.lower() for gpu in gpuhunt.KNOWN_TENSTORRENT_ACCELERATORS}
 _BIND_ADDRESS_ARG = "bind_address"
 
 logger = get_logger(__name__)
@@ -350,6 +350,7 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
         if gpu_spec.count.max == 0:
             return
         has_amd_gpu: bool
+        has_tt_gpu: bool
         vendor = gpu_spec.vendor
         if vendor is None:
             names = gpu_spec.name
@@ -362,6 +363,8 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
                         vendors.add(gpuhunt.AcceleratorVendor.NVIDIA)
                     elif name in _KNOWN_AMD_GPUS:
                         vendors.add(gpuhunt.AcceleratorVendor.AMD)
+                    elif name in _KNOWN_TENSTORRENT_GPUS:
+                        vendors.add(gpuhunt.AcceleratorVendor.TENSTORRENT)
                     else:
                         maybe_tpu_version, _, maybe_tpu_cores = name.partition("-")
                         if maybe_tpu_version in _KNOWN_TPU_VERSIONS and maybe_tpu_cores.isdigit():
@@ -380,15 +383,22 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
                 # to execute a run on an instance with an AMD accelerator with a default
                 # CUDA image, not a big deal.
                 has_amd_gpu = gpuhunt.AcceleratorVendor.AMD in vendors
+                has_tt_gpu = gpuhunt.AcceleratorVendor.TENSTORRENT in vendors
             else:
                 # If neither gpu.vendor nor gpu.name is set, assume Nvidia.
                 vendor = gpuhunt.AcceleratorVendor.NVIDIA
                 has_amd_gpu = False
+                has_tt_gpu = False
             gpu_spec.vendor = vendor
         else:
             has_amd_gpu = vendor == gpuhunt.AcceleratorVendor.AMD
+            has_tt_gpu = vendor == gpuhunt.AcceleratorVendor.TENSTORRENT
         if has_amd_gpu and conf.image is None:
-            raise ConfigurationError("`image` is required if `resources.gpu.vendor` is AMD.")
+            raise ConfigurationError("`image` is required if `resources.gpu.vendor` is `amd`")
+        if has_tt_gpu and conf.image is None:
+            raise ConfigurationError(
+                "`image` is required if `resources.gpu.vendor` is `tenstorrent`"
+            )
 
 
 class RunWithPortsConfigurator(BaseRunConfigurator):

dstack/_internal/cli/utils/fleet.py

@@ -79,7 +79,9 @@ def get_fleets_table(
                 "BACKEND": backend,
                 "REGION": region,
                 "RESOURCES": resources,
-                "PRICE": f"${instance.price:.4}" if instance.price is not None else "",
+                "PRICE": f"${instance.price:.4f}".rstrip("0").rstrip(".")
+                if instance.price is not None
+                else "",
                 "STATUS": status,
                 "CREATED": format_date(instance.created),
                 "ERROR": error,

dstack/_internal/cli/utils/run.py

@@ -1,3 +1,4 @@
+import os
 from typing import Any, Dict, List, Optional, Union
 
 from rich.markup import escape
@@ -36,7 +37,7 @@ def print_run_plan(
 
     req = job_plan.job_spec.requirements
     pretty_req = req.pretty_format(resources_only=True)
-    max_price = f"${req.max_price:g}" if req.max_price else "-"
+    max_price = f"${req.max_price:3f}".rstrip("0").rstrip(".") if req.max_price else "-"
     max_duration = (
         format_pretty_duration(job_plan.job_spec.max_duration)
         if job_plan.job_spec.max_duration
@@ -94,14 +95,12 @@ def print_run_plan(
             props.add_row(th("Inactivity duration"), inactivity_duration)
     props.add_row(th("Reservation"), run_spec.configuration.reservation or "-")
 
-    offers = Table(box=None)
+    offers = Table(box=None, expand=os.get_terminal_size()[0] <= 110)
     offers.add_column("#")
-    offers.add_column("BACKEND")
-    offers.add_column("REGION")
-    offers.add_column("INSTANCE TYPE")
-    offers.add_column("RESOURCES")
-    offers.add_column("SPOT")
-    offers.add_column("PRICE")
+    offers.add_column("BACKEND", style="grey58", ratio=2)
+    offers.add_column("RESOURCES", ratio=4)
+    offers.add_column("INSTANCE TYPE", style="grey58", no_wrap=True, ratio=2)
+    offers.add_column("PRICE", style="grey58", ratio=1)
     offers.add_column()
 
     job_plan.offers = job_plan.offers[:max_offers] if max_offers else job_plan.offers
@@ -122,14 +121,12 @@ def print_run_plan(
             instance += f" ({offer.blocks}/{offer.total_blocks})"
         offers.add_row(
             f"{i}",
-            offer.backend.replace("remote", "ssh"),
-            offer.region,
+            offer.backend.replace("remote", "ssh") + " (" + offer.region + ")",
+            r.pretty_format(include_spot=True),
             instance,
-            r.pretty_format(),
-            "yes" if r.spot else "no",
-            f"${offer.price:g}",
+            f"${offer.price:.4f}".rstrip("0").rstrip("."),
             availability,
-            style=None if i == 1 else "secondary",
+            style=None if i == 1 or not include_run_properties else "secondary",
         )
     if job_plan.total_offers > len(job_plan.offers):
         offers.add_row("", "...", style="secondary")
@@ -141,7 +138,8 @@ def print_run_plan(
         if job_plan.total_offers > len(job_plan.offers):
             console.print(
                 f"[secondary] Shown {len(job_plan.offers)} of {job_plan.total_offers} offers, "
-                f"${job_plan.max_price:g} max[/]"
+                f"${job_plan.max_price:3f}".rstrip("0").rstrip(".")
+                + "max[/]"
             )
         console.print()
     else:
@@ -151,19 +149,18 @@ def print_run_plan(
 def get_runs_table(
     runs: List[Run], verbose: bool = False, format_date: DateFormatter = pretty_date
 ) -> Table:
-    table = Table(box=None)
-    table.add_column("NAME", style="bold", no_wrap=True)
-    table.add_column("BACKEND", style="grey58")
+    table = Table(box=None, expand=os.get_terminal_size()[0] <= 110)
+    table.add_column("NAME", style="bold", no_wrap=True, ratio=2)
+    table.add_column("BACKEND", style="grey58", ratio=2)
+    table.add_column("RESOURCES", ratio=3 if not verbose else 2)
     if verbose:
-        table.add_column("INSTANCE", no_wrap=True)
-    table.add_column("RESOURCES")
+        table.add_column("INSTANCE", no_wrap=True, ratio=1)
+        table.add_column("RESERVATION", no_wrap=True, ratio=1)
+    table.add_column("PRICE", style="grey58", ratio=1)
+    table.add_column("STATUS", no_wrap=True, ratio=1)
+    table.add_column("SUBMITTED", style="grey58", no_wrap=True, ratio=1)
     if verbose:
-        table.add_column("RESERVATION", no_wrap=True)
-    table.add_column("PRICE", no_wrap=True)
-    table.add_column("STATUS", no_wrap=True)
-    table.add_column("SUBMITTED", style="grey58", no_wrap=True)
-    if verbose:
-        table.add_column("ERROR", no_wrap=True)
+        table.add_column("ERROR", no_wrap=True, ratio=2)
 
     for run in runs:
         run_error = _get_run_error(run)
@@ -202,10 +199,10 @@ def get_runs_table(
                 job_row.update(
                     {
                         "BACKEND": f"{jpd.backend.value.replace('remote', 'ssh')} ({jpd.region})",
-                        "INSTANCE": instance,
                         "RESOURCES": resources.pretty_format(include_spot=True),
+                        "INSTANCE": instance,
                         "RESERVATION": jpd.reservation,
-                        "PRICE": f"${jpd.price:.4}",
+                        "PRICE": f"${jpd.price:.4f}".rstrip("0").rstrip("."),
                     }
                 )
             if len(run.jobs) == 1:

dstack/_internal/core/backends/gcp/resources.py

@@ -205,12 +205,17 @@ def _get_network_interfaces(
     else:
         network_interface.access_configs = []
 
+    if extra_subnetworks:
+        # Multiple interfaces are set only for GPU VM that require gVNIC for best performance
+        network_interface.nic_type = compute_v1.NetworkInterface.NicType.GVNIC.name
+
     network_interfaces = [network_interface]
     for network, subnetwork in extra_subnetworks or []:
         network_interfaces.append(
             compute_v1.NetworkInterface(
                 network=network,
                 subnetwork=subnetwork,
+                nic_type=compute_v1.NetworkInterface.NicType.GVNIC.name,
             )
         )
     return network_interfaces
@@ -437,7 +442,7 @@ def wait_for_operation(operation: Operation, verbose_name: str = "operation", ti
         raise
     except Exception as e:
         # Write only debug logs here.
-        # The unexpected errors will be propagated and logged appropriatly by the caller.
+        # The unexpected errors will be propagated and logged appropriately by the caller.
         logger.debug("Error during %s: %s", verbose_name, e)
         raise operation.exception() or e
     return result

dstack/_internal/core/backends/vastai/compute.py

@@ -43,7 +43,8 @@ class VastAICompute(Compute):
                     "reliability2": {"gte": 0.9},
                     "inet_down": {"gt": 128},
                     "verified": {"eq": True},
-                    "cuda_max_good": {"gte": 11.8},
+                    "cuda_max_good": {"gte": 12.1},
+                    "compute_cap": {"gte": 600},
                 }
             )
         )

dstack/_internal/core/errors.py

@@ -22,6 +22,10 @@ class URLNotFoundError(ClientError):
     pass
 
 
+class MethodNotAllowedError(ClientError):
+    pass
+
+
 class ServerClientErrorCode(str, enum.Enum):
     UNSPECIFIED_ERROR = "error"
     RESOURCE_EXISTS = "resource_exists"

dstack/_internal/core/models/fleets.py

@@ -269,6 +269,8 @@ class FleetSpec(CoreModel):
     configuration_path: Optional[str] = None
     profile: Profile
     autocreated: bool = False
+    # merged_profile stores profile parameters merged from profile and configuration.
+    # Read profile parameters from merged_profile instead of profile directly.
     # TODO: make merged_profile a computed field after migrating to pydanticV2
     merged_profile: Annotated[Profile, Field(exclude=True)] = None
 

dstack/_internal/core/models/instances.py

@@ -57,7 +57,7 @@ class Resources(CoreModel):
         if self.memory_mib > 0:
             resources["memory"] = f"{self.memory_mib / 1024:.0f}GB"
         if self.disk.size_mib > 0:
-            resources["disk_size"] = f"{self.disk.size_mib / 1024:.1f}GB"
+            resources["disk_size"] = f"{self.disk.size_mib / 1024:.0f}GB"
         if self.gpus:
             gpu = self.gpus[0]
             resources["gpu_name"] = gpu.name
@@ -66,7 +66,7 @@ class Resources(CoreModel):
                 resources["gpu_memory"] = f"{gpu.memory_mib / 1024:.0f}GB"
         output = pretty_resources(**resources)
         if include_spot and self.spot:
-            output += ", SPOT"
+            output += " (spot)"
         return output
 
 

dstack/_internal/core/models/resources.py

@@ -246,6 +246,8 @@ class GPUSpec(CoreModel):
         v = v.lower()
         if v == "tpu":
             return gpuhunt.AcceleratorVendor.GOOGLE
+        if v == "tt":
+            return gpuhunt.AcceleratorVendor.TENSTORRENT
         return gpuhunt.AcceleratorVendor.cast(v)
 
 

dstack/_internal/core/models/runs.py

@@ -162,7 +162,7 @@ class Requirements(CoreModel):
             if self.spot is not None:
                 res += f", {'spot' if self.spot else 'on-demand'}"
             if self.max_price is not None:
-                res += f" under ${self.max_price:g} per hour"
+                res += f" under ${self.max_price:3f}".rstrip("0").rstrip(".") + " per hour"
         return res
 
 
@@ -357,6 +357,8 @@ class RunSpec(CoreModel):
             description="The contents of the SSH public key that will be used to connect to the run."
         ),
     ]
+    # merged_profile stores profile parameters merged from profile and configuration.
+    # Read profile parameters from merged_profile instead of profile directly.
     # TODO: make merged_profile a computed field after migrating to pydanticV2
     merged_profile: Annotated[Profile, Field(exclude=True)] = None
 

dstack/_internal/server/background/tasks/process_submitted_jobs.py

@@ -197,7 +197,7 @@ async def _process_submitted_job(session: AsyncSession, job_model: JobModel):
         pool_instances = list(res.unique().scalars().all())
         instances_ids = sorted([i.id for i in pool_instances])
         if get_db().dialect_name == "sqlite":
-            # Start new transaction to see commited changes after lock
+            # Start new transaction to see committed changes after lock
             await session.commit()
         async with get_locker().lock_ctx(InstanceModel.__tablename__, instances_ids):
             # If another job freed the instance but is still trying to detach volumes,

dstack/_internal/server/routers/gateways.py

@@ -47,9 +47,10 @@ async def create_gateway(
     session: AsyncSession = Depends(get_session),
     user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectAdmin()),
 ) -> models.Gateway:
-    _, project = user_project
+    user, project = user_project
     return await gateways.create_gateway(
         session=session,
+        user=user,
         project=project,
         configuration=body.configuration,
     )

dstack/_internal/server/services/config.py

@@ -29,6 +29,7 @@ from dstack._internal.server.services.permissions import (
     DefaultPermissions,
     set_default_permissions,
 )
+from dstack._internal.server.services.plugins import load_plugins
 from dstack._internal.utils.logging import get_logger
 
 logger = get_logger(__name__)
@@ -38,7 +39,7 @@ logger = get_logger(__name__)
 # If a collection has nested collections, it will be assigned the block style. Otherwise it will have the flow style.
 #
 # We want mapping to always be displayed in block-style but lists without nested objects in flow-style.
-# So we define a custom representeter
+# So we define a custom representer.
 
 
 def seq_representer(dumper, sequence):
@@ -75,7 +76,10 @@ class ServerConfig(CoreModel):
     ] = None
     default_permissions: Annotated[
         Optional[DefaultPermissions], Field(description="The default user permissions")
-    ]
+    ] = None
+    plugins: Annotated[
+        Optional[List[str]], Field(description="The server-side plugins to enable")
+    ] = None
 
 
 class ServerConfigManager:
@@ -112,6 +116,7 @@ class ServerConfigManager:
             await self._apply_project_config(
                 session=session, owner=owner, project_config=project_config
             )
+        load_plugins(enabled_plugins=self.config.plugins or [])
 
     async def _apply_project_config(
         self,

dstack/_internal/server/services/fleets.py

@@ -55,6 +55,7 @@ from dstack._internal.server.services.locking import (
     get_locker,
     string_to_lock_id,
 )
+from dstack._internal.server.services.plugins import apply_plugin_policies
 from dstack._internal.server.services.projects import (
     get_member,
     get_member_permissions,
@@ -234,7 +235,14 @@ async def get_plan(
     user: UserModel,
     spec: FleetSpec,
 ) -> FleetPlan:
+    # Spec must be copied by parsing to calculate merged_profile
     effective_spec = FleetSpec.parse_obj(spec.dict())
+    effective_spec = apply_plugin_policies(
+        user=user.name,
+        project=project.name,
+        spec=effective_spec,
+    )
+    effective_spec = FleetSpec.parse_obj(effective_spec.dict())
     current_fleet: Optional[Fleet] = None
     current_fleet_id: Optional[uuid.UUID] = None
     if effective_spec.configuration.name is not None:
@@ -330,6 +338,13 @@ async def create_fleet(
     user: UserModel,
     spec: FleetSpec,
 ) -> Fleet:
+    # Spec must be copied by parsing to calculate merged_profile
+    spec = apply_plugin_policies(
+        user=user.name,
+        project=project.name,
+        spec=spec,
+    )
+    spec = FleetSpec.parse_obj(spec.dict())
     _validate_fleet_spec(spec)
 
     if spec.configuration.ssh_config is not None:

dstack/_internal/server/services/gateways/__init__.py

@@ -31,13 +31,19 @@ from dstack._internal.core.models.gateways import (
     Gateway,
     GatewayComputeConfiguration,
     GatewayConfiguration,
+    GatewaySpec,
     GatewayStatus,
     LetsEncryptGatewayCertificate,
 )
 from dstack._internal.core.services import validate_dstack_resource_name
 from dstack._internal.server import settings
 from dstack._internal.server.db import get_db
-from dstack._internal.server.models import GatewayComputeModel, GatewayModel, ProjectModel
+from dstack._internal.server.models import (
+    GatewayComputeModel,
+    GatewayModel,
+    ProjectModel,
+    UserModel,
+)
 from dstack._internal.server.services.backends import (
     check_backend_type_available,
     get_project_backend_by_type_or_error,
@@ -50,6 +56,7 @@ from dstack._internal.server.services.locking import (
     get_locker,
     string_to_lock_id,
 )
+from dstack._internal.server.services.plugins import apply_plugin_policies
 from dstack._internal.server.utils.common import gather_map_async
 from dstack._internal.utils.common import get_current_datetime, run_async
 from dstack._internal.utils.crypto import generate_rsa_key_pair_bytes
@@ -129,9 +136,17 @@ async def create_gateway_compute(
 
 async def create_gateway(
     session: AsyncSession,
+    user: UserModel,
     project: ProjectModel,
     configuration: GatewayConfiguration,
 ) -> Gateway:
+    spec = apply_plugin_policies(
+        user=user.name,
+        project=project.name,
+        # Create pseudo spec until the gateway API is updated to accept spec
+        spec=GatewaySpec(configuration=configuration),
+    )
+    configuration = spec.configuration
     _validate_gateway_configuration(configuration)
 
     backend_model, _ = await get_project_backend_with_model_by_type_or_error(
@@ -140,7 +155,7 @@ async def create_gateway(
 
     lock_namespace = f"gateway_names_{project.name}"
     if get_db().dialect_name == "sqlite":
-        # Start new transaction to see commited changes after lock
+        # Start new transaction to see committed changes after lock
         await session.commit()
     elif get_db().dialect_name == "postgresql":
         await session.execute(

dstack/_internal/server/services/plugins.py

@@ -0,0 +1,77 @@
+import itertools
+from importlib import import_module
+
+from backports.entry_points_selectable import entry_points  # backport for Python 3.9
+
+from dstack._internal.core.errors import ServerClientError
+from dstack._internal.utils.logging import get_logger
+from dstack.plugins import ApplyPolicy, ApplySpec, Plugin
+
+logger = get_logger(__name__)
+
+
+_PLUGINS: list[Plugin] = []
+
+
+def load_plugins(enabled_plugins: list[str]):
+    _PLUGINS.clear()
+    plugins_entrypoints = entry_points(group="dstack.plugins")
+    plugins_to_load = enabled_plugins.copy()
+    for entrypoint in plugins_entrypoints:
+        if entrypoint.name not in enabled_plugins:
+            logger.info(
+                ("Found not enabled plugin %s. Plugin will not be loaded."),
+                entrypoint.name,
+            )
+            continue
+        try:
+            module_path, _, class_name = entrypoint.value.partition(":")
+            module = import_module(module_path)
+        except ImportError:
+            logger.warning(
+                (
+                    "Failed to load plugin %s when importing %s."
+                    " Ensure the module is on the import path."
+                ),
+                entrypoint.name,
+                entrypoint.value,
+            )
+            continue
+        plugin_class = getattr(module, class_name, None)
+        if plugin_class is None:
+            logger.warning(
+                ("Failed to load plugin %s: plugin class %s not found in module %s."),
+                entrypoint.name,
+                class_name,
+                module_path,
+            )
+            continue
+        if not issubclass(plugin_class, Plugin):
+            logger.warning(
+                ("Failed to load plugin %s: plugin class %s is not a subclass of Plugin."),
+                entrypoint.name,
+                class_name,
+            )
+            continue
+        plugins_to_load.remove(entrypoint.name)
+        _PLUGINS.append(plugin_class())
+        logger.info("Loaded plugin %s", entrypoint.name)
+    if plugins_to_load:
+        logger.warning("Enabled plugins not found: %s", plugins_to_load)
+
+
+def apply_plugin_policies(user: str, project: str, spec: ApplySpec) -> ApplySpec:
+    policies = _get_apply_policies()
+    for policy in policies:
+        try:
+            spec = policy.on_apply(user=user, project=project, spec=spec)
+        except ValueError as e:
+            msg = None
+            if len(e.args) > 0:
+                msg = e.args[0]
+            raise ServerClientError(msg)
+    return spec
+
+
+def _get_apply_policies() -> list[ApplyPolicy]:
+    return list(itertools.chain(*[p.get_apply_policies() for p in _PLUGINS]))

dstack/_internal/server/services/runs.py

@@ -79,6 +79,7 @@ from dstack._internal.server.services.jobs import (
 from dstack._internal.server.services.locking import get_locker, string_to_lock_id
 from dstack._internal.server.services.logging import fmt
 from dstack._internal.server.services.offers import get_offers_by_requirements
+from dstack._internal.server.services.plugins import apply_plugin_policies
 from dstack._internal.server.services.projects import list_project_models, list_user_project_models
 from dstack._internal.server.services.users import get_user_model_by_name
 from dstack._internal.utils.logging import get_logger
@@ -279,7 +280,14 @@ async def get_plan(
     run_spec: RunSpec,
     max_offers: Optional[int],
 ) -> RunPlan:
+    # Spec must be copied by parsing to calculate merged_profile
     effective_run_spec = RunSpec.parse_obj(run_spec.dict())
+    effective_run_spec = apply_plugin_policies(
+        user=user.name,
+        project=project.name,
+        spec=effective_run_spec,
+    )
+    effective_run_spec = RunSpec.parse_obj(effective_run_spec.dict())
     _validate_run_spec_and_set_defaults(effective_run_spec)
 
     profile = effective_run_spec.merged_profile
@@ -370,28 +378,36 @@ async def apply_plan(
     plan: ApplyRunPlanInput,
     force: bool,
 ) -> Run:
-    _validate_run_spec_and_set_defaults(plan.run_spec)
-    if plan.run_spec.run_name is None:
+    run_spec = plan.run_spec
+    run_spec = apply_plugin_policies(
+        user=user.name,
+        project=project.name,
+        spec=run_spec,
+    )
+    # Spec must be copied by parsing to calculate merged_profile
+    run_spec = RunSpec.parse_obj(run_spec.dict())
+    _validate_run_spec_and_set_defaults(run_spec)
+    if run_spec.run_name is None:
         return await submit_run(
             session=session,
             user=user,
             project=project,
-            run_spec=plan.run_spec,
+            run_spec=run_spec,
         )
     current_resource = await get_run_by_name(
         session=session,
         project=project,
-        run_name=plan.run_spec.run_name,
+        run_name=run_spec.run_name,
     )
     if current_resource is None or current_resource.status.is_finished():
         return await submit_run(
             session=session,
             user=user,
             project=project,
-            run_spec=plan.run_spec,
+            run_spec=run_spec,
         )
     try:
-        _check_can_update_run_spec(current_resource.run_spec, plan.run_spec)
+        _check_can_update_run_spec(current_resource.run_spec, run_spec)
     except ServerClientError:
         # The except is only needed to raise an appropriate error if run is active
         if not current_resource.status.is_finished():
@@ -409,14 +425,12 @@ async def apply_plan(
     # FIXME: potentially long write transaction
     # Avoid getting run_model after update
     await session.execute(
-        update(RunModel)
-        .where(RunModel.id == current_resource.id)
-        .values(run_spec=plan.run_spec.json())
+        update(RunModel).where(RunModel.id == current_resource.id).values(run_spec=run_spec.json())
     )
     run = await get_run_by_name(
         session=session,
         project=project,
-        run_name=plan.run_spec.run_name,
+        run_name=run_spec.run_name,
     )
     return common_utils.get_or_error(run)
 
@@ -436,7 +450,7 @@ async def submit_run(
 
     lock_namespace = f"run_names_{project.name}"
     if get_db().dialect_name == "sqlite":
-        # Start new transaction to see commited changes after lock
+        # Start new transaction to see committed changes after lock
         await session.commit()
     elif get_db().dialect_name == "postgresql":
         await session.execute(

dstack/_internal/server/services/volumes.py

@@ -21,6 +21,7 @@ from dstack._internal.core.models.volumes import (
     VolumeConfiguration,
     VolumeInstance,
     VolumeProvisioningData,
+    VolumeSpec,
     VolumeStatus,
 )
 from dstack._internal.core.services import validate_dstack_resource_name
@@ -38,6 +39,7 @@ from dstack._internal.server.services.locking import (
     get_locker,
     string_to_lock_id,
 )
+from dstack._internal.server.services.plugins import apply_plugin_policies
 from dstack._internal.server.services.projects import list_project_models, list_user_project_models
 from dstack._internal.utils import common, random_names
 from dstack._internal.utils.logging import get_logger
@@ -203,11 +205,18 @@ async def create_volume(
     user: UserModel,
     configuration: VolumeConfiguration,
 ) -> Volume:
+    spec = apply_plugin_policies(
+        user=user.name,
+        project=project.name,
+        # Create pseudo spec until the volume API is updated to accept spec
+        spec=VolumeSpec(configuration=configuration),
+    )
+    configuration = spec.configuration
     _validate_volume_configuration(configuration)
 
     lock_namespace = f"volume_names_{project.name}"
     if get_db().dialect_name == "sqlite":
-        # Start new transaction to see commited changes after lock
+        # Start new transaction to see committed changes after lock
         await session.commit()
     elif get_db().dialect_name == "postgresql":
         await session.execute(

dstack/_internal/utils/common.py

@@ -110,25 +110,26 @@ def pretty_resources(
     """
     parts = []
     if cpus is not None:
-        parts.append(f"{cpus}xCPU")
+        parts.append(f"cpu={cpus}")
     if memory is not None:
-        parts.append(f"{memory}")
+        parts.append(f"mem={memory}")
+    if disk_size:
+        parts.append(f"disk={disk_size}")
     if gpu_count:
         gpu_parts = []
+        gpu_parts.append(f"{gpu_name or 'gpu'}")
         if gpu_memory is not None:
             gpu_parts.append(f"{gpu_memory}")
+        if gpu_count is not None:
+            gpu_parts.append(f"{gpu_count}")
         if total_gpu_memory is not None:
-            gpu_parts.append(f"total {total_gpu_memory}")
+            gpu_parts.append(f"{total_gpu_memory}")
         if compute_capability is not None:
             gpu_parts.append(f"{compute_capability}")
 
-        gpu = f"{gpu_count}x{gpu_name or 'GPU'}"
-        if gpu_parts:
-            gpu += f" ({', '.join(gpu_parts)})"
+        gpu = ":".join(gpu_parts)
         parts.append(gpu)
-    if disk_size:
-        parts.append(f"{disk_size} (disk)")
-    return ", ".join(parts)
+    return " ".join(parts)
 
 
 def since(timestamp: str) -> datetime:

dstack/api/server/__init__.py

@@ -6,7 +6,12 @@ from typing import Dict, List, Optional, Type
 import requests
 
 from dstack import version
-from dstack._internal.core.errors import ClientError, ServerClientError, URLNotFoundError
+from dstack._internal.core.errors import (
+    ClientError,
+    MethodNotAllowedError,
+    ServerClientError,
+    URLNotFoundError,
+)
 from dstack._internal.utils.logging import get_logger
 from dstack.api.server._backends import BackendsAPIClient
 from dstack.api.server._fleets import FleetsAPIClient
@@ -156,6 +161,8 @@ class APIClient:
                 )
             if resp.status_code == 404:
                 raise URLNotFoundError(f"Status code 404 when requesting {resp.request.url}")
+            if resp.status_code == 405:
+                raise MethodNotAllowedError(f"Status code 405 when requesting {resp.request.url}")
             if 400 <= resp.status_code < 600:
                 raise ClientError(
                     f"Unexpected error: status code {resp.status_code}"

dstack/plugins/__init__.py

@@ -0,0 +1,8 @@
+# ruff: noqa: F401
+from dstack._internal.core.models.fleets import FleetSpec
+from dstack._internal.core.models.gateways import GatewaySpec
+from dstack._internal.core.models.runs import RunSpec
+from dstack._internal.core.models.volumes import VolumeSpec
+from dstack.plugins._base import ApplyPolicy, Plugin
+from dstack.plugins._models import ApplySpec
+from dstack.plugins._utils import get_plugin_logger

dstack/plugins/_base.py

@@ -0,0 +1,72 @@
+from dstack._internal.core.models.fleets import FleetSpec
+from dstack._internal.core.models.gateways import GatewaySpec
+from dstack._internal.core.models.runs import RunSpec
+from dstack._internal.core.models.volumes import VolumeSpec
+from dstack.plugins._models import ApplySpec
+
+
+class ApplyPolicy:
+    """
+    A base apply policy class to modify specs on `dstack apply`.
+    Subclass it and return the subclass instance in `Plugin.get_apply_policies()`.
+    """
+
+    def on_apply(self, user: str, project: str, spec: ApplySpec) -> ApplySpec:
+        """
+        Modify `spec` before it's applied.
+        Raise `ValueError` for `spec` to be rejected as invalid.
+
+        This method can be called twice:
+          * first when a user gets a plan
+          * second when a user applies a plan
+
+        In both cases, the original spec is passed, so the method does not
+        need to check if it modified the spec before.
+
+        It's safe to modify and return `spec` without copying.
+        """
+        if isinstance(spec, RunSpec):
+            return self.on_run_apply(user=user, project=project, spec=spec)
+        if isinstance(spec, FleetSpec):
+            return self.on_fleet_apply(user=user, project=project, spec=spec)
+        if isinstance(spec, VolumeSpec):
+            return self.on_volume_apply(user=user, project=project, spec=spec)
+        if isinstance(spec, GatewaySpec):
+            return self.on_gateway_apply(user=user, project=project, spec=spec)
+        raise ValueError(f"Unknown spec type {type(spec)}")
+
+    def on_run_apply(self, user: str, project: str, spec: RunSpec) -> RunSpec:
+        """
+        Called by the default `on_apply()` implementation for runs.
+        """
+        return spec
+
+    def on_fleet_apply(self, user: str, project: str, spec: FleetSpec) -> FleetSpec:
+        """
+        Called by the default `on_apply()` implementation for fleets.
+        """
+        return spec
+
+    def on_volume_apply(self, user: str, project: str, spec: VolumeSpec) -> VolumeSpec:
+        """
+        Called by the default `on_apply()` implementation for volumes.
+        """
+        return spec
+
+    def on_gateway_apply(self, user: str, project: str, spec: GatewaySpec) -> GatewaySpec:
+        """
+        Called by the default `on_apply()` implementation for gateways.
+        """
+        return spec
+
+
+class Plugin:
+    """
+    A base plugin class.
+    Plugins must subclass it, implement public methods,
+    and register the subclass as an entrypoint of the package
+    (https://packaging.python.org/en/latest/specifications/entry-points/).
+    """
+
+    def get_apply_policies(self) -> list[ApplyPolicy]:
+        return []

dstack/plugins/_models.py

@@ -0,0 +1,8 @@
+from typing import TypeVar
+
+from dstack._internal.core.models.fleets import FleetSpec
+from dstack._internal.core.models.gateways import GatewaySpec
+from dstack._internal.core.models.runs import RunSpec
+from dstack._internal.core.models.volumes import VolumeSpec
+
+ApplySpec = TypeVar("ApplySpec", RunSpec, FleetSpec, VolumeSpec, GatewaySpec)

dstack/plugins/_utils.py

@@ -0,0 +1,19 @@
+import logging
+
+from dstack._internal.utils.logging import get_logger
+
+
+def get_plugin_logger(name: str) -> logging.Logger:
+    """
+    Use this function to set up loggers in plugins.
+
+    Put at the top of the plugin modules:
+
+    ```
+    from dstack.plugins import get_plugin_logger
+
+    logger = get_plugin_logger(__name__)
+    ```
+
+    """
+    return get_logger(f"dstack.plugins.{name}")

dstack/version.py

@@ -1,3 +1,3 @@
-__version__ = "0.19.6rc1"
+__version__ = "0.19.7"
 __is_release__ = True
 base_image = "0.7"

{dstack-0.19.6rc1.dist-info → dstack-0.19.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dstack
-Version: 0.19.6rc1
+Version: 0.19.7
 Summary: dstack is an open-source orchestration engine for running AI workloads on any cloud or on-premises.
 Project-URL: Homepage, https://dstack.ai
 Project-URL: Source, https://github.com/dstackai/dstack
@@ -21,7 +21,7 @@ Requires-Dist: cryptography
 Requires-Dist: cursor
 Requires-Dist: filelock
 Requires-Dist: gitpython
-Requires-Dist: gpuhunt<0.2.0,>=0.1.3
+Requires-Dist: gpuhunt==0.1.5
 Requires-Dist: jsonschema
 Requires-Dist: packaging
 Requires-Dist: paramiko>=3.2.0
@@ -52,6 +52,7 @@ Requires-Dist: azure-mgmt-compute>=29.1.0; extra == 'all'
 Requires-Dist: azure-mgmt-network<28.0.0,>=23.0.0; extra == 'all'
 Requires-Dist: azure-mgmt-resource>=22.0.0; extra == 'all'
 Requires-Dist: azure-mgmt-subscription>=3.1.1; extra == 'all'
+Requires-Dist: backports-entry-points-selectable; extra == 'all'
 Requires-Dist: boto3; extra == 'all'
 Requires-Dist: botocore; extra == 'all'
 Requires-Dist: datacrunch; extra == 'all'
@@ -87,6 +88,7 @@ Requires-Dist: alembic-postgresql-enum; extra == 'aws'
 Requires-Dist: alembic>=1.10.2; extra == 'aws'
 Requires-Dist: apscheduler<4; extra == 'aws'
 Requires-Dist: asyncpg; extra == 'aws'
+Requires-Dist: backports-entry-points-selectable; extra == 'aws'
 Requires-Dist: boto3; extra == 'aws'
 Requires-Dist: botocore; extra == 'aws'
 Requires-Dist: docker>=6.0.0; extra == 'aws'
@@ -117,6 +119,7 @@ Requires-Dist: azure-mgmt-compute>=29.1.0; extra == 'azure'
 Requires-Dist: azure-mgmt-network<28.0.0,>=23.0.0; extra == 'azure'
 Requires-Dist: azure-mgmt-resource>=22.0.0; extra == 'azure'
 Requires-Dist: azure-mgmt-subscription>=3.1.1; extra == 'azure'
+Requires-Dist: backports-entry-points-selectable; extra == 'azure'
 Requires-Dist: docker>=6.0.0; extra == 'azure'
 Requires-Dist: fastapi; extra == 'azure'
 Requires-Dist: grpcio>=1.50; extra == 'azure'
@@ -139,6 +142,7 @@ Requires-Dist: alembic-postgresql-enum; extra == 'datacrunch'
 Requires-Dist: alembic>=1.10.2; extra == 'datacrunch'
 Requires-Dist: apscheduler<4; extra == 'datacrunch'
 Requires-Dist: asyncpg; extra == 'datacrunch'
+Requires-Dist: backports-entry-points-selectable; extra == 'datacrunch'
 Requires-Dist: datacrunch; extra == 'datacrunch'
 Requires-Dist: docker>=6.0.0; extra == 'datacrunch'
 Requires-Dist: fastapi; extra == 'datacrunch'
@@ -170,6 +174,7 @@ Requires-Dist: alembic-postgresql-enum; extra == 'gcp'
 Requires-Dist: alembic>=1.10.2; extra == 'gcp'
 Requires-Dist: apscheduler<4; extra == 'gcp'
 Requires-Dist: asyncpg; extra == 'gcp'
+Requires-Dist: backports-entry-points-selectable; extra == 'gcp'
 Requires-Dist: docker>=6.0.0; extra == 'gcp'
 Requires-Dist: fastapi; extra == 'gcp'
 Requires-Dist: google-api-python-client>=2.80.0; extra == 'gcp'
@@ -199,6 +204,7 @@ Requires-Dist: alembic-postgresql-enum; extra == 'kubernetes'
 Requires-Dist: alembic>=1.10.2; extra == 'kubernetes'
 Requires-Dist: apscheduler<4; extra == 'kubernetes'
 Requires-Dist: asyncpg; extra == 'kubernetes'
+Requires-Dist: backports-entry-points-selectable; extra == 'kubernetes'
 Requires-Dist: docker>=6.0.0; extra == 'kubernetes'
 Requires-Dist: fastapi; extra == 'kubernetes'
 Requires-Dist: grpcio>=1.50; extra == 'kubernetes'
@@ -222,6 +228,7 @@ Requires-Dist: alembic-postgresql-enum; extra == 'lambda'
 Requires-Dist: alembic>=1.10.2; extra == 'lambda'
 Requires-Dist: apscheduler<4; extra == 'lambda'
 Requires-Dist: asyncpg; extra == 'lambda'
+Requires-Dist: backports-entry-points-selectable; extra == 'lambda'
 Requires-Dist: boto3; extra == 'lambda'
 Requires-Dist: botocore; extra == 'lambda'
 Requires-Dist: docker>=6.0.0; extra == 'lambda'
@@ -246,6 +253,7 @@ Requires-Dist: alembic-postgresql-enum; extra == 'nebius'
 Requires-Dist: alembic>=1.10.2; extra == 'nebius'
 Requires-Dist: apscheduler<4; extra == 'nebius'
 Requires-Dist: asyncpg; extra == 'nebius'
+Requires-Dist: backports-entry-points-selectable; extra == 'nebius'
 Requires-Dist: docker>=6.0.0; extra == 'nebius'
 Requires-Dist: fastapi; extra == 'nebius'
 Requires-Dist: grpcio>=1.50; extra == 'nebius'
@@ -269,6 +277,7 @@ Requires-Dist: alembic-postgresql-enum; extra == 'oci'
 Requires-Dist: alembic>=1.10.2; extra == 'oci'
 Requires-Dist: apscheduler<4; extra == 'oci'
 Requires-Dist: asyncpg; extra == 'oci'
+Requires-Dist: backports-entry-points-selectable; extra == 'oci'
 Requires-Dist: docker>=6.0.0; extra == 'oci'
 Requires-Dist: fastapi; extra == 'oci'
 Requires-Dist: grpcio>=1.50; extra == 'oci'
@@ -292,6 +301,7 @@ Requires-Dist: alembic-postgresql-enum; extra == 'server'
 Requires-Dist: alembic>=1.10.2; extra == 'server'
 Requires-Dist: apscheduler<4; extra == 'server'
 Requires-Dist: asyncpg; extra == 'server'
+Requires-Dist: backports-entry-points-selectable; extra == 'server'
 Requires-Dist: docker>=6.0.0; extra == 'server'
 Requires-Dist: fastapi; extra == 'server'
 Requires-Dist: grpcio>=1.50; extra == 'server'

{dstack-0.19.6rc1.dist-info → dstack-0.19.7.dist-info}/RECORD

@@ -1,5 +1,5 @@
 dstack/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/version.py,sha256=UUpgUhdND2BCWQQIg6bu0Vl3VRKsFMFZ9MFjakM6j9w,67
+dstack/version.py,sha256=ikdWwuUYtHtNaJ5eN6aIj36R0uhZ5NlYDVtxl3E_02M,64
 dstack/_internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/compat.py,sha256=bF9U9fTMfL8UVhCouedoUSTYFl7UAOiU0WXrnRoByxw,40
 dstack/_internal/settings.py,sha256=otvcNT0X5UnGZdoNIWNFZBohQRzLme9Zc6oiBzc1BEk,796
@@ -29,21 +29,21 @@ dstack/_internal/cli/services/profile.py,sha256=zfLRuesv5DSnLkbOU-TouvYmGGaKvw33
 dstack/_internal/cli/services/repos.py,sha256=ImJuElGjsR4OQ7dBquy4PCQUbBM4bR4vhyLwMrVBS7k,3307
 dstack/_internal/cli/services/configurators/__init__.py,sha256=z94VPBFqybP8Zpwy3CzYxmpPAqYBOvRRLpXoz2H4GKI,2697
 dstack/_internal/cli/services/configurators/base.py,sha256=bGfde2zoma28lLE8MUACO4-NKT1CdJJQJoXrzjpz0mQ,3360
-dstack/_internal/cli/services/configurators/fleet.py,sha256=gxGmAI-03jk1EXwQpABMWf4DedqWnYRqJWXmvrL6ty4,13972
+dstack/_internal/cli/services/configurators/fleet.py,sha256=jm4tNH6QQVplLdboCTlvRYUee3nZ0UYb_qLTrvtYVYM,14049
 dstack/_internal/cli/services/configurators/gateway.py,sha256=czB2s89s7IowOmWnpDwWErPAUlW3FvFMizImhrkQiBM,8927
-dstack/_internal/cli/services/configurators/run.py,sha256=7oUFcoIj4aPfJoI-o5wd4JqYrm5zsdFgQ5OHPiP4wrA,23092
+dstack/_internal/cli/services/configurators/run.py,sha256=arYV5pDAtA4ZJSwY1y4OOYeJpi4JLDteHM2ubI6dCaw,23710
 dstack/_internal/cli/services/configurators/volume.py,sha256=riMXLQbgvHIIFwLKdHfad-_0iE9wE3G_rUmXU5P3ZS8,8519
 dstack/_internal/cli/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/cli/utils/common.py,sha256=rfmzqrsgR3rXW3wj0vxDdvrhUUg2aIy4A6E9MZbd55g,1763
-dstack/_internal/cli/utils/fleet.py,sha256=NpqTdcMVcuUGke9drroxDoNnmifbKV7iziqdGPLc3lk,3885
+dstack/_internal/cli/utils/fleet.py,sha256=ch-LN1X9boSm-rFLW4mAJRmz0XliLhH0LvKD2DqSt2g,3942
 dstack/_internal/cli/utils/gateway.py,sha256=qMYa1NTAT_O98x2_mSyWDRbiHj5fqt6xUXFh9NIUwAM,1502
 dstack/_internal/cli/utils/rich.py,sha256=Gx1MJU929kMKsbdo9qF7XHARNta2426Ssb-xMLVhwbQ,5710
-dstack/_internal/cli/utils/run.py,sha256=N3S0GG0kQI8dCn5XQDNKNjxOaAFHZG6EJ-RLCYYhHow,8617
+dstack/_internal/cli/utils/run.py,sha256=30K28Y8fnbcME7kVZv8iflpH4FyyC50XehrKLYvMYZ4,8965
 dstack/_internal/cli/utils/updates.py,sha256=sAPYYptkFzQnGaRjv7FV7HOj-Be3IXGe63xj-sVEpv4,2566
 dstack/_internal/cli/utils/volume.py,sha256=mU9I06dVMFbpjfkefxrZNoSWadKLoib3U14rHudNQN4,1975
 dstack/_internal/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/core/consts.py,sha256=c1Yd5UY6Qx7KeuYgloXWncWhMsYj6TqwlElda7NtB98,254
-dstack/_internal/core/errors.py,sha256=zoYS3na_2qB33TRFYOKTv41XbeQsLqY6HCCpOkgAFVI,3260
+dstack/_internal/core/errors.py,sha256=tJcZXwrvWvvtzIqkzLxJJMrhDuAQl8eOk0xpKCqqHzw,3313
 dstack/_internal/core/backends/__init__.py,sha256=fwgV8CN8Ap6MZmWklMGHHf0roliBtqne-ijjVOpWcgc,2467
 dstack/_internal/core/backends/configurators.py,sha256=JxGfZwcmL90akMFkAzzZ_fzPvU2No0pBaBqU_g0D-y0,3775
 dstack/_internal/core/backends/models.py,sha256=aKQOrDEStouuwY4MacSen7SkoyAa6HR6a6PFq5-cbNk,4088
@@ -87,7 +87,7 @@ dstack/_internal/core/backends/gcp/backend.py,sha256=OvTv1c7j4LTPCIEtkwD3-q6Eo1Q
 dstack/_internal/core/backends/gcp/compute.py,sha256=uDPoDoqx6S29bAZBQk-Bwoz_u6oMfQB8u2FpMLZ-uDg,40815
 dstack/_internal/core/backends/gcp/configurator.py,sha256=mvI7WMz8cC1YnN-0KFIIEqkfcEBehRJI0WgKnqILjv0,6730
 dstack/_internal/core/backends/gcp/models.py,sha256=biLA3rlFcoPatAZpKycuIl-8PdnNSAFiDCJjov65_zo,4612
-dstack/_internal/core/backends/gcp/resources.py,sha256=MueDEZwn7K-55Gbq9X9UZBPLd313XSFJwzT7htqfo6U,16208
+dstack/_internal/core/backends/gcp/resources.py,sha256=gLoRo9Z1--n55JTEDVaofUsBo-8h5Nfbhsb_2ydvmK8,16487
 dstack/_internal/core/backends/gcp/features/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/core/backends/gcp/features/tcpx.py,sha256=8bDR5kwF5qke5EWNdBscdbZQnC7oVXKSls3WPcoXgZI,2902
 dstack/_internal/core/backends/kubernetes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -142,7 +142,7 @@ dstack/_internal/core/backends/tensordock/models.py,sha256=oxbSWRCsD0_j4-2lQYo4L
 dstack/_internal/core/backends/vastai/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/core/backends/vastai/api_client.py,sha256=b-qgx352lnxhBXichJtURFGxR7Jd9JYx4WXaxYV-HH0,4940
 dstack/_internal/core/backends/vastai/backend.py,sha256=NawGHNYVmgNwL8HruEA84rimnpeSyXccdJXr6Dr95UY,566
-dstack/_internal/core/backends/vastai/compute.py,sha256=I0TMGGEh2J0LIIzO1hYRWXDgSELC4eV2F8zczSLHtno,4862
+dstack/_internal/core/backends/vastai/compute.py,sha256=ZjeJF196lXTjO5gQ8fnQtvNEVZdYpcVODmZOOV3g96Q,4911
 dstack/_internal/core/backends/vastai/configurator.py,sha256=mn8Sx0ZFgTlBa2mf2xNjGymXhbfswgHTjYDUnTiI028,2267
 dstack/_internal/core/backends/vastai/models.py,sha256=2xpFE-ouYUBienxZXam2Mo4dy9enSv-FuVJXeYnjVLk,1036
 dstack/_internal/core/backends/vultr/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -156,16 +156,16 @@ dstack/_internal/core/models/common.py,sha256=XWd79dmFGMrdpTcStH5fVmNXCKE0s7FsIo
 dstack/_internal/core/models/config.py,sha256=JJ7rT7dztzTWCY5TkoyxXxTvG5D4IFYhGe7EzwkLOWQ,581
 dstack/_internal/core/models/configurations.py,sha256=c2717pYpna6u2bmtmC3DlX67HtQa-Tud387mUVa4vEg,18498
 dstack/_internal/core/models/envs.py,sha256=yq84YRFBILOy4x3XnGcTgYpbZ69eFTCQPgBCr9Ndov4,4969
-dstack/_internal/core/models/fleets.py,sha256=o6Kgx7hh0CpGew1h0GIflWmTvSvW6vF1o53g2ccpDJQ,11894
+dstack/_internal/core/models/fleets.py,sha256=gbP2rj1ODDuy4IfTbPNB-ae050thvzTtA8uDb5iTmdY,12059
 dstack/_internal/core/models/gateways.py,sha256=_O8EWwHWLdgNoWY4P4u71KM-uEr5DDp42LXfyv1qMDI,4054
-dstack/_internal/core/models/instances.py,sha256=nX48UuSK6LccFBZ0QSzKWGLuGJp5POmX5HyI93yBXpI,5392
+dstack/_internal/core/models/instances.py,sha256=gwJFF0wJhPQCd0Y5ZiFD9e4_EADP3ktHMurxGV0QR18,5393
 dstack/_internal/core/models/logs.py,sha256=Lsmtd_NrnChMjBJahUZpFb1j8Xobix9FHWf1L47FOGs,443
 dstack/_internal/core/models/metrics.py,sha256=Xb8hCXUL-ncQ3PMsErIUAJTe9gwh5jyrQ4UQoZbibsc,269
 dstack/_internal/core/models/placement.py,sha256=WJVq5ENJykyRarQzL2EeYQag_9_jV7VSAtR_xoFvPVM,720
 dstack/_internal/core/models/profiles.py,sha256=seeysTuMv1vVUmpHAZgrMUGcbMtH7hSMFIvfx0Qk__0,10406
 dstack/_internal/core/models/projects.py,sha256=H5ZZRiyUEKifpTFAhl45KBi5ly7ooE0WmI329myK360,643
-dstack/_internal/core/models/resources.py,sha256=28xfFnTd5FHjI-WOtRlIj6L4Hmh0sQAMpIMo8nVe7LE,11301
-dstack/_internal/core/models/runs.py,sha256=sIuAkBegFsUZIHJPQHd6ZlYxi22bzpQzz_2jxtjfuLs,18196
+dstack/_internal/core/models/resources.py,sha256=xvPuRlF4y3Pj_HNBJTmmzuroTggQQsaR87KERu6asN0,11380
+dstack/_internal/core/models/runs.py,sha256=RvUJHIP1h_GAYiqvTwJxnW46AipEsM92pB33cv21Mn4,18391
 dstack/_internal/core/models/secrets.py,sha256=IQyemsNpSzqOCB-VlVTuc4gyPFmXXO4mhko0Ur0ey3I,221
 dstack/_internal/core/models/server.py,sha256=Hkc1v2s3KOiwslsWVmhUOAzcSeREoG-HD1SzSX9WUGg,152
 dstack/_internal/core/models/services.py,sha256=2Hpi7j0Q1shaf_0wd0C0044AJAmuYi-D3qx3PH849oI,3076
@@ -262,7 +262,7 @@ dstack/_internal/server/background/tasks/process_placement_groups.py,sha256=FqGf
 dstack/_internal/server/background/tasks/process_prometheus_metrics.py,sha256=u8hCXjOOek7VLEsmLy2VnDXFmIwTNjrJwcpWG7a1zW0,5093
 dstack/_internal/server/background/tasks/process_running_jobs.py,sha256=U6JdkEnpIApbiSRLKxqjNwA9WFAZY2zZNXujofhUd_g,34719
 dstack/_internal/server/background/tasks/process_runs.py,sha256=EI1W6HUyB-og3g8BDP_GsBrJjQ-Z3JvZHTuJf7CRKRM,17974
-dstack/_internal/server/background/tasks/process_submitted_jobs.py,sha256=ypZw75ZK6_vnp-434obnkFVj6Yll9UylOOydAUrWpc4,26602
+dstack/_internal/server/background/tasks/process_submitted_jobs.py,sha256=-XOApBgmn9ZyCoeXgnbp6cnsFT3uxE_-xqLtn1ez5dc,26603
 dstack/_internal/server/background/tasks/process_terminating_jobs.py,sha256=0Z3Q409RwSxOL_pgK8JktBthjtESEUH3ahwTLsTdYPk,3800
 dstack/_internal/server/background/tasks/process_volumes.py,sha256=206rbT4ICeZtEmqh_94Rry_fgHfFLLaSEX9W-svwFk4,5089
 dstack/_internal/server/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -336,7 +336,7 @@ dstack/_internal/server/migrations/versions/ffa99edd1988_add_jobterminationreaso
 dstack/_internal/server/routers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/server/routers/backends.py,sha256=vSCP-wbH-fsoKDR1TrMyQCz5UA-biiY0Lo7ERVz_L_g,4684
 dstack/_internal/server/routers/fleets.py,sha256=ikXeJ7phoVocU3i-_S6V5RIMygjh-KHuhEIAS5Cw1wo,5565
-dstack/_internal/server/routers/gateways.py,sha256=am5HnyWUydgUbEjIdjYBcob8CEIrx4bRtdj9kQhutkA,3149
+dstack/_internal/server/routers/gateways.py,sha256=K_VG5Dt_2F_cykWjtFj4GBwtm_D3xfsE3Bf-lDgQx_w,3171
 dstack/_internal/server/routers/instances.py,sha256=XOogTC9My2Zv0ck37_PbHKoZI-j4QeGrP2sN5wpX7Ow,1579
 dstack/_internal/server/routers/logs.py,sha256=_Euk283LbhlwHibJTKM-7YcpbeQFtWBqMfbOry3PSkU,1159
 dstack/_internal/server/routers/metrics.py,sha256=VFgWhkOvxVFDLlRM_kXHYFylLcfCD6UjXInvcd7H4dY,2314
@@ -365,9 +365,9 @@ dstack/_internal/server/schemas/volumes.py,sha256=9iwaQLMhA6aj9XmtdU_9jWVhpzNOtF
 dstack/_internal/server/security/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/server/security/permissions.py,sha256=FJ_8YPhjmebA4jQjtQoAGEaj1Hahb_po0tYRCQ18aaE,4940
 dstack/_internal/server/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/_internal/server/services/config.py,sha256=X61ypyuBQqAuWi-mVFJ7Uk1DyGfHqk8_d2C6xIG0yxE,10434
+dstack/_internal/server/services/config.py,sha256=yo8njslwfS7_blhbPPhOtzCMyg8N_mmFSw5aPvirSzw,10691
 dstack/_internal/server/services/docker.py,sha256=zAvjFHxIP03Td92NzbGEScz0piLjloE60tQ7vz0AeCA,5328
-dstack/_internal/server/services/fleets.py,sha256=za87tcoGYxQAMjfTPSf4cBsmNHjzMFyYDltsrtlXcK0,26517
+dstack/_internal/server/services/fleets.py,sha256=JQuXwunED_m_gvLeAO6cUVxeeRKTFAlBHNqQKuvLUZ8,27076
 dstack/_internal/server/services/instances.py,sha256=omCzPWEIRJQx7WkwyJPmxMtkjas9KtgIFl-zLtnurBY,18610
 dstack/_internal/server/services/locking.py,sha256=7JUgNSplKRx7dxC4LIpmWw81agUtslEDTeDiNMPbAVg,3013
 dstack/_internal/server/services/logging.py,sha256=Nu1628kW2hqB__N0Eyr07wGWjVWxfyJnczonTJ72kSM,417
@@ -375,13 +375,14 @@ dstack/_internal/server/services/metrics.py,sha256=jKLy1jSCVR_crqVu_CmsOMbvMkucW
 dstack/_internal/server/services/offers.py,sha256=At_fRbFCEHBsiJiqIZMMowofO4o0pfs6tSOtFgZc8Nc,7086
 dstack/_internal/server/services/permissions.py,sha256=l7Ngdelmn65vjw13NcOdaC6lBYMRuSw6FbHzYwdK3nE,1005
 dstack/_internal/server/services/placement.py,sha256=DWZ8-iAE3o0J0xaHikuJYZzpuBiq7lj41LiAP1PfoEs,1773
+dstack/_internal/server/services/plugins.py,sha256=e5ESJVw5EaxkmxLXdvZdIGaFNxy8k5QOiPQD2zN9w84,2650
 dstack/_internal/server/services/projects.py,sha256=Je1iWZ-ArmyFxK1yMUzod5WRXyiIDxtuVp6pHcdctTQ,14988
 dstack/_internal/server/services/prometheus.py,sha256=xq5G-Q2BJup9lS2F6__0wUVTs-k1Gr3dYclGzo2WoWo,12474
 dstack/_internal/server/services/repos.py,sha256=f9ztN7jz_2gvD9hXF5sJwWDVyG2-NHRfjIdSukowPh8,9342
-dstack/_internal/server/services/runs.py,sha256=7CcA41wCtso1Ao7d9g1OEs146FlB3kedCJXcRJU3Jqk,37265
+dstack/_internal/server/services/runs.py,sha256=W8MuRchSGijmFZo7TCdm2ko0LnICUGc1jtOz0qt5p5E,37824
 dstack/_internal/server/services/storage.py,sha256=6I0xI_3_RpJNbKZwHjDnjrEwXGdHfiaeb5li15T-M1I,1884
 dstack/_internal/server/services/users.py,sha256=W-5xL7zsHNjeG7BBK54RWGvIrBOrw-FF0NcG_z9qhoE,7466
-dstack/_internal/server/services/volumes.py,sha256=sdjSobFdY7CpB5LzZmD6oG1FfmXoF-j283hLCqowFgU,15883
+dstack/_internal/server/services/volumes.py,sha256=vfKY6eZp64I58Mfdvrk9Wig7deveD2Rw4ET1cbc1Sog,16238
 dstack/_internal/server/services/backends/__init__.py,sha256=Aqo1GoqhZ_FsLEkCcBrvReKSq6E5w4QbBLrDXfGjiKU,13154
 dstack/_internal/server/services/backends/handlers.py,sha256=j-MhBxrpdepoDG7f2tApjFnE23RVO5I15-hxHyOWnew,3251
 dstack/_internal/server/services/encryption/__init__.py,sha256=3kCw_cxC3-Un1OIofdW5Gqsm0ZCXXTlGz09cULBx_uc,3155
@@ -389,7 +390,7 @@ dstack/_internal/server/services/encryption/keys/__init__.py,sha256=47DEQpj8HBSa
 dstack/_internal/server/services/encryption/keys/aes.py,sha256=2He1p2_Rg6hnCeLIGJo-Yfdsij7so_338oY49RXuL3Q,2276
 dstack/_internal/server/services/encryption/keys/base.py,sha256=mqumJiidoexUPoqxhQG6J_SpC1WGYwkdjKm1MUWnXo8,352
 dstack/_internal/server/services/encryption/keys/identity.py,sha256=ryb_YSV6u4c7W1OsVfEpzJvZCrR4zZYlzLw_GpjpD2Q,741
-dstack/_internal/server/services/gateways/__init__.py,sha256=sFP2vyXvFrg1mqr399s30r8IFMymewXw4NrIjhMyExU,21119
+dstack/_internal/server/services/gateways/__init__.py,sha256=Up8uFsEQDBE0yOXn7n5o7Q8MkY0XJfbMWViMFd2EIL4,21530
 dstack/_internal/server/services/gateways/client.py,sha256=XIJX3fGBbZ_AG8qZMTSE8KAB_ojq5YJFa0OXoD_dofg,7493
 dstack/_internal/server/services/gateways/connection.py,sha256=ot3lV85XdmCT45vBWeyj57nLPcLPNm316zu3jMyeWjA,5625
 dstack/_internal/server/services/gateways/pool.py,sha256=0LclTl1tyx-doS78LeaAKjr-SMp98zuwh5f9s06JSd0,1914
@@ -535,7 +536,7 @@ dstack/_internal/server/utils/common.py,sha256=PbjXtqYy1taKXpyG5ys8cIrz9MXqc9dBA
 dstack/_internal/server/utils/logging.py,sha256=bxUS2uWG5snypNRfL0d5sMLCDytyOZac81PSQlb7_rs,1907
 dstack/_internal/server/utils/routers.py,sha256=OzL9Oxy-1no7Txk1r-Pvf28l3S25CYJlyAscYY345Xg,4729
 dstack/_internal/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/_internal/utils/common.py,sha256=z6ZCQnvxOfZoiXx7hRKljMsHt5ZDfhNdeyH9RyJ2v3I,8789
+dstack/_internal/utils/common.py,sha256=1Ig2D3XUpYfiDHAsqC3JRKwx8PsWp1sWLRVHOlEKwwU,8827
 dstack/_internal/utils/crypto.py,sha256=2RTSyzePuwwqc1X2HO6lwcSFyZ2kujnqluoICQ2DLJQ,1462
 dstack/_internal/utils/dxf.py,sha256=wguK9s6-69kqSDZkxd1kFEr6VlH5ixvFRJxizyOuJ8I,3229
 dstack/_internal/utils/env.py,sha256=HRbIspHpKHh05fMZeV23-hrZoV6vVMuniefD08u6ey0,357
@@ -558,7 +559,7 @@ dstack/api/_public/backends.py,sha256=w_cIUVU3L1tN8VoPH6-ltq-oJewlfXWTiEFXfDvm-1
 dstack/api/_public/repos.py,sha256=2ufUtxY8WyPci9tCGYzsnyIFTXNJfTuPzDYvpTydZ6k,6133
 dstack/api/_public/runs.py,sha256=tILFldJb4EKMYYBMr5rUiD5aJGIPB4-eLXzwwcqPbVg,27828
 dstack/api/huggingface/__init__.py,sha256=oIrEij3wttLZ1yrywEGvCMd6zswMQrX5pPjrqdSi0UA,2201
-dstack/api/server/__init__.py,sha256=pkWTwAuR4SVx8zo6GNER0ERqEUcr67vljtr0ZbxV4tQ,5914
+dstack/api/server/__init__.py,sha256=Zyl1M51tifn4pB150yIsh39N96qUgMjg5XplcElHDxg,6097
 dstack/api/server/_backends.py,sha256=tSvJ4j-yp-S-4IYo7pKHluDaSsx6Xbwo08Ff6Do85fo,1639
 dstack/api/server/_fleets.py,sha256=CMxBH49WF18urYx5kj4-B0D5a26QbaO2uDr5Xg-HH0M,4480
 dstack/api/server/_gateways.py,sha256=FAjSi1l6dj1H0mHuFwcCH-QbRxAxVzPI8xE2cmcvWqo,2688
@@ -573,8 +574,12 @@ dstack/api/server/_users.py,sha256=XzhgGKc5Tsr0-xkz3T6rGyWZ1tO7aYNhLux2eE7dAoY,1
 dstack/api/server/_volumes.py,sha256=xxOt8o5G-bhMh6wSvF4BDFNoqVEhlM4BXQr2KvX0pN0,1937
 dstack/api/server/utils.py,sha256=i1KX4CNXVeDj9CnytdzsJz0bxjvvfLRTb7xw8oqtEtQ,1040
 dstack/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack-0.19.6rc1.dist-info/METADATA,sha256=fXqtx7wlO0dEoAV7Aerh3ExEO9ZdwU0j15HT6Kio8Fw,19377
-dstack-0.19.6rc1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-dstack-0.19.6rc1.dist-info/entry_points.txt,sha256=GnLrMS8hx3rWAySQjA7tPNhtixV6a-brRkmal1PKoHc,58
-dstack-0.19.6rc1.dist-info/licenses/LICENSE.md,sha256=qDABaRGjSKVOib1U8viw2P_96sIK7Puo426784oD9f8,15976
-dstack-0.19.6rc1.dist-info/RECORD,,
+dstack/plugins/__init__.py,sha256=buT1pcyORLgVbl89ATkRWJPhvejriVz7sNBjvuZRCRE,403
+dstack/plugins/_base.py,sha256=-etiB-EozaJCg2wtmONfj8ic-K03qXvXyl_TIDp-kNE,2662
+dstack/plugins/_models.py,sha256=1Gw--mDQ1_0FFr9Zur9LE8UbMoWESUpTdHHt12AyIZo,341
+dstack/plugins/_utils.py,sha256=FqeWYb7zOrgZkO9Bd8caL5I81_TUEsysIzvxsULrmzk,392
+dstack-0.19.7.dist-info/METADATA,sha256=g67K4f_PhyA2NrwUmQ-SG1cb45HpHMoQUtd5YQwZ3d4,20042
+dstack-0.19.7.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+dstack-0.19.7.dist-info/entry_points.txt,sha256=GnLrMS8hx3rWAySQjA7tPNhtixV6a-brRkmal1PKoHc,58
+dstack-0.19.7.dist-info/licenses/LICENSE.md,sha256=qDABaRGjSKVOib1U8viw2P_96sIK7Puo426784oD9f8,15976
+dstack-0.19.7.dist-info/RECORD,,