dstack 0.18.44__py3-none-any.whl → 0.19.0rc1__py3-none-any.whl

dstack/_internal/server/services/metrics.py

@@ -7,8 +7,11 @@ from typing import Optional
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from dstack._internal.core.models.instances import Resources
 from dstack._internal.core.models.metrics import JobMetrics, Metric
 from dstack._internal.server.models import JobMetricsPoint, JobModel
+from dstack._internal.server.services.jobs import get_job_provisioning_data, get_job_runtime_data
+from dstack._internal.utils.common import get_or_error
 from dstack._internal.utils.logging import get_logger
 
 logger = get_logger(__name__)
@@ -47,10 +50,10 @@ async def get_job_metrics(
     # we need at least 2 points to calculate cpu_usage_percent
     if len(points) < 2:
         return JobMetrics(metrics=[])
-    return _calculate_job_metrics(points)
+    return _calculate_job_metrics(job_model, points)
 
 
-def _calculate_job_metrics(points: Sequence[JobMetricsPoint]) -> JobMetrics:
+def _calculate_job_metrics(job_model: JobModel, points: Sequence[JobMetricsPoint]) -> JobMetrics:
     timestamps: list[datetime] = []
     cpu_usage_points: list[int] = []
     memory_usage_points: list[int] = []
@@ -58,6 +61,23 @@ def _calculate_job_metrics(points: Sequence[JobMetricsPoint]) -> JobMetrics:
     gpus_memory_usage_points: defaultdict[int, list[int]] = defaultdict(list)
     gpus_util_points: defaultdict[int, list[int]] = defaultdict(list)
 
+    cpus_detected_num: Optional[int] = None
+    memory_total: Optional[int] = None
+    gpu_memory_total: Optional[int] = None
+    resources: Optional[Resources] = None
+    jrd = get_job_runtime_data(job_model)
+    if jrd is not None and jrd.offer is not None:
+        resources = jrd.offer.instance.resources
+    else:
+        jpd = get_job_provisioning_data(job_model)
+        if jpd is not None:
+            resources = jpd.instance_type.resources
+    if resources is not None:
+        cpus_detected_num = resources.cpus
+        memory_total = resources.memory_mib * 1024 * 1024
+        if len(resources.gpus) > 0:
+            gpu_memory_total = resources.gpus[0].memory_mib * 1024 * 1024
+
     gpus_detected_num: Optional[int] = None
     gpus_detected_num_mismatch: bool = False
     for point, prev_point in zip(points, points[1:]):
@@ -93,6 +113,10 @@ def _calculate_job_metrics(points: Sequence[JobMetricsPoint]) -> JobMetrics:
             values=memory_working_set_points,
         ),
     ]
+    if cpus_detected_num is not None:
+        metrics.append(_make_constant_metric("cpus_detected_num", timestamps, cpus_detected_num))
+    if memory_total is not None:
+        metrics.append(_make_constant_metric("memory_total_bytes", timestamps, memory_total))
     if gpus_detected_num_mismatch:
         # If number of GPUs changed in the time window, skip GPU metrics altogether, otherwise
         # results can be unpredictable (e.g, one GPU takes place of another, as they are
@@ -100,18 +124,12 @@ def _calculate_job_metrics(points: Sequence[JobMetricsPoint]) -> JobMetrics:
         logger.warning("gpus_detected_num mismatch, skipping GPU metrics")
     else:
         metrics.append(
-            # As gpus_detected_num expected to be constant, we add only two points — the latest
-            # and the earliest in the batch
-            Metric(
-                name="gpus_detected_num",
-                timestamps=[timestamps[0], timestamps[-1]]
-                if len(timestamps) > 1
-                else [timestamps[0]],
-                values=[gpus_detected_num, gpus_detected_num]
-                if len(timestamps) > 1
-                else [gpus_detected_num],
-            )
+            _make_constant_metric("gpus_detected_num", timestamps, get_or_error(gpus_detected_num))
         )
+        if gpu_memory_total is not None:
+            metrics.append(
+                _make_constant_metric("gpu_memory_total_bytes", timestamps, gpu_memory_total)
+            )
         for index, gpu_memory_usage_points in gpus_memory_usage_points.items():
             metrics.append(
                 Metric(
@@ -131,6 +149,14 @@ def _calculate_job_metrics(points: Sequence[JobMetricsPoint]) -> JobMetrics:
     return JobMetrics(metrics=metrics)
 
 
+def _make_constant_metric(name: str, timestamps: list[datetime], value: float) -> Metric:
+    return Metric(
+        name=name,
+        timestamps=timestamps,
+        values=[value] * len(timestamps),
+    )
+
+
 def _get_cpu_usage(last_point: JobMetricsPoint, prev_point: JobMetricsPoint) -> int:
     window = last_point.timestamp_micro - prev_point.timestamp_micro
     if window == 0:

dstack/_internal/server/services/offers.py

@@ -7,7 +7,7 @@ from dstack._internal.core.backends import (
     BACKENDS_WITH_MULTINODE_SUPPORT,
     BACKENDS_WITH_RESERVATION_SUPPORT,
 )
-from dstack._internal.core.backends.base import Backend
+from dstack._internal.core.backends.base.backend import Backend
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.instances import (
     InstanceOfferWithAvailability,

dstack/_internal/server/services/projects.py

@@ -7,19 +7,22 @@ from sqlalchemy import func as safunc
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import joinedload
 
-from dstack._internal.core.errors import ForbiddenError, ResourceExistsError, ServerClientError
-from dstack._internal.core.models.backends import BackendInfo
-from dstack._internal.core.models.backends.dstack import (
-    DstackBaseBackendConfigInfo,
-    DstackConfigInfo,
+from dstack._internal.core.backends.configurators import get_configurator
+from dstack._internal.core.backends.dstack.models import (
+    DstackBackendConfig,
+    DstackBaseBackendConfig,
 )
+from dstack._internal.core.backends.models import BackendInfo
+from dstack._internal.core.errors import ForbiddenError, ResourceExistsError, ServerClientError
 from dstack._internal.core.models.common import is_core_model_instance
 from dstack._internal.core.models.projects import Member, MemberPermissions, Project
 from dstack._internal.core.models.users import GlobalRole, ProjectRole
 from dstack._internal.server.models import MemberModel, ProjectModel, UserModel
 from dstack._internal.server.schemas.projects import MemberSetting
 from dstack._internal.server.services import users
-from dstack._internal.server.services.backends import get_configurator
+from dstack._internal.server.services.backends import (
+    get_backend_config_from_backend_model,
+)
 from dstack._internal.server.services.permissions import get_default_permissions
 from dstack._internal.server.settings import DEFAULT_PROJECT_NAME
 from dstack._internal.utils.common import get_current_datetime, run_async
@@ -176,12 +179,16 @@ async def set_project_members(
     # FIXME: potentially long write transaction
     # clear_project_members() issues DELETE without commit
     await clear_project_members(session=session, project=project)
-    usernames = [m.username for m in members]
-    res = await session.execute(select(UserModel).where(UserModel.name.in_(usernames)))
+    names = [m.username for m in members]
+    res = await session.execute(
+        select(UserModel).where((UserModel.name.in_(names)) | (UserModel.email.in_(names)))
+    )
     users = res.scalars().all()
+    # Create lookup maps for both username and email
     username_to_user = {user.name: user for user in users}
+    email_to_user = {user.email: user for user in users if user.email}
     for i, member in enumerate(members):
-        user_to_add = username_to_user.get(member.username)
+        user_to_add = username_to_user.get(member.username) or email_to_user.get(member.username)
         if user_to_add is None:
             continue
         await add_project_member(
@@ -376,20 +383,22 @@ def project_model_to_project(
                     b.type.value,
                 )
                 continue
-            config_info = configurator.get_config_info(model=b, include_creds=False)
-            if is_core_model_instance(config_info, DstackConfigInfo):
-                for backend_type in config_info.base_backends:
+            backend_config = get_backend_config_from_backend_model(
+                configurator, b, include_creds=False
+            )
+            if is_core_model_instance(backend_config, DstackBackendConfig):
+                for backend_type in backend_config.base_backends:
                     backends.append(
                         BackendInfo(
                             name=backend_type,
-                            config=DstackBaseBackendConfigInfo(type=backend_type),
+                            config=DstackBaseBackendConfig(type=backend_type),
                         )
                     )
             else:
                 backends.append(
                     BackendInfo(
                         name=b.type,
-                        config=config_info,
+                        config=backend_config,
                     )
                 )
     return Project(

dstack/_internal/server/services/prometheus.py

@@ -1,4 +1,6 @@
+import itertools
 from collections.abc import Generator, Iterable
+from datetime import timezone
 
 from prometheus_client import Metric
 from prometheus_client.parser import text_string_to_metric_families
@@ -7,21 +9,172 @@ from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import joinedload
 
-from dstack._internal.core.models.runs import JobStatus
-from dstack._internal.server.models import JobModel, JobPrometheusMetrics, ProjectModel
+from dstack._internal.core.models.instances import InstanceStatus
+from dstack._internal.core.models.runs import JobStatus, RunSpec
+from dstack._internal.server.models import (
+    InstanceModel,
+    JobModel,
+    JobPrometheusMetrics,
+    ProjectModel,
+    RunModel,
+)
+from dstack._internal.server.services.instances import get_instance_offer
+from dstack._internal.server.services.jobs import get_job_provisioning_data, get_job_runtime_data
+from dstack._internal.utils.common import get_current_datetime
+
+_INSTANCE_DURATION = "dstack_instance_duration_seconds_total"
+_INSTANCE_PRICE = "dstack_instance_price_dollars_per_hour"
+_INSTANCE_GPU_COUNT = "dstack_instance_gpu_count"
+_JOB_DURATION = "dstack_job_duration_seconds_total"
+_JOB_PRICE = "dstack_job_price_dollars_per_hour"
+_JOB_GPU_COUNT = "dstack_job_gpu_count"
 
 
 async def get_metrics(session: AsyncSession) -> str:
+    metrics_iter = itertools.chain(
+        await get_instance_metrics(session),
+        await get_job_metrics(session),
+        await get_job_gpu_metrics(session),
+    )
+    return "\n".join(_render_metrics(metrics_iter)) + "\n"
+
+
+async def get_instance_metrics(session: AsyncSession) -> Iterable[Metric]:
+    res = await session.execute(
+        select(InstanceModel)
+        .join(ProjectModel)
+        .where(
+            InstanceModel.deleted == False,
+            InstanceModel.status.in_(
+                [
+                    InstanceStatus.PROVISIONING,
+                    InstanceStatus.IDLE,
+                    InstanceStatus.BUSY,
+                    InstanceStatus.TERMINATING,
+                ]
+            ),
+        )
+        .order_by(ProjectModel.name, InstanceModel.name)
+        .options(
+            joinedload(InstanceModel.project),
+            joinedload(InstanceModel.fleet),
+        )
+    )
+    instances = res.unique().scalars().all()
+    metrics: dict[str, Metric] = {
+        _INSTANCE_DURATION: Metric(
+            name=_INSTANCE_DURATION,
+            documentation="Total seconds the instance is running",
+            typ="counter",
+        ),
+        _INSTANCE_PRICE: Metric(
+            name=_INSTANCE_PRICE, documentation="Instance price, USD/hour", typ="gauge"
+        ),
+        _INSTANCE_GPU_COUNT: Metric(
+            name=_INSTANCE_GPU_COUNT, documentation="Instance GPU count", typ="gauge"
+        ),
+    }
+    now = get_current_datetime()
+    for instance in instances:
+        fleet = instance.fleet
+        offer = get_instance_offer(instance)
+        gpu = ""
+        gpu_count = 0
+        if offer is not None and len(offer.instance.resources.gpus) > 0:
+            gpu = offer.instance.resources.gpus[0].name
+            gpu_count = len(offer.instance.resources.gpus)
+        labels: dict[str, str] = {
+            "dstack_project_name": instance.project.name,
+            "dstack_fleet_name": fleet.name if fleet is not None else "",
+            "dstack_fleet_id": str(fleet.id) if fleet is not None else "",
+            "dstack_instance_name": str(instance.name),
+            "dstack_instance_id": str(instance.id),
+            "dstack_instance_type": offer.instance.name if offer is not None else "",
+            "dstack_backend": instance.backend.value if instance.backend is not None else "",
+            "dstack_gpu": gpu,
+        }
+        duration = (now - instance.created_at.replace(tzinfo=timezone.utc)).total_seconds()
+        metrics[_INSTANCE_DURATION].add_sample(
+            name=_INSTANCE_DURATION, labels=labels, value=duration
+        )
+        metrics[_INSTANCE_PRICE].add_sample(
+            name=_INSTANCE_PRICE, labels=labels, value=instance.price or 0.0
+        )
+        metrics[_INSTANCE_GPU_COUNT].add_sample(
+            name=_INSTANCE_GPU_COUNT, labels=labels, value=gpu_count
+        )
+    return metrics.values()
+
+
+async def get_job_metrics(session: AsyncSession) -> Iterable[Metric]:
+    res = await session.execute(
+        select(JobModel)
+        .join(ProjectModel)
+        .where(
+            JobModel.status.in_(
+                [
+                    JobStatus.PROVISIONING,
+                    JobStatus.PULLING,
+                    JobStatus.RUNNING,
+                    JobStatus.TERMINATING,
+                ]
+            )
+        )
+        .order_by(ProjectModel.name, JobModel.job_name)
+        .options(
+            joinedload(JobModel.project),
+            joinedload(JobModel.run).joinedload(RunModel.user),
+        )
+    )
+    jobs = res.scalars().all()
+    metrics: dict[str, Metric] = {
+        _JOB_DURATION: Metric(
+            name=_JOB_DURATION, documentation="Total seconds the job is running", typ="counter"
+        ),
+        _JOB_PRICE: Metric(
+            name=_JOB_PRICE, documentation="Job instance price, USD/hour", typ="gauge"
+        ),
+        _JOB_GPU_COUNT: Metric(name=_JOB_GPU_COUNT, documentation="Job GPU count", typ="gauge"),
+    }
+    now = get_current_datetime()
+    for job in jobs:
+        jpd = get_job_provisioning_data(job)
+        if jpd is None:
+            continue
+        jrd = get_job_runtime_data(job)
+        gpus = jpd.instance_type.resources.gpus
+        price = jpd.price
+        if jrd is not None and jrd.offer is not None:
+            gpus = jrd.offer.instance.resources.gpus
+            price = jrd.offer.price
+        run_spec = RunSpec.__response__.parse_raw(job.run.run_spec)
+        labels = _get_job_labels(job)
+        labels["dstack_run_type"] = run_spec.configuration.type
+        labels["dstack_backend"] = jpd.get_base_backend().value
+        labels["dstack_gpu"] = gpus[0].name if gpus else ""
+        duration = (now - job.submitted_at.replace(tzinfo=timezone.utc)).total_seconds()
+        metrics[_JOB_DURATION].add_sample(name=_JOB_DURATION, labels=labels, value=duration)
+        metrics[_JOB_PRICE].add_sample(name=_JOB_PRICE, labels=labels, value=price)
+        metrics[_JOB_GPU_COUNT].add_sample(name=_JOB_GPU_COUNT, labels=labels, value=len(gpus))
+    return metrics.values()
+
+
+async def get_job_gpu_metrics(session: AsyncSession) -> Iterable[Metric]:
     res = await session.execute(
         select(JobPrometheusMetrics)
         .join(JobModel)
         .join(ProjectModel)
         .where(JobModel.status.in_([JobStatus.RUNNING]))
         .order_by(ProjectModel.name, JobModel.job_name)
-        .options(joinedload(JobPrometheusMetrics.job).joinedload(JobModel.project))
+        .options(
+            joinedload(JobPrometheusMetrics.job).joinedload(JobModel.project),
+            joinedload(JobPrometheusMetrics.job)
+            .joinedload(JobModel.run)
+            .joinedload(RunModel.user),
+        )
     )
     metrics_models = res.scalars().all()
-    return _process_metrics(metrics_models)
+    return _parse_and_enrich_job_gpu_metrics(metrics_models)
 
 
 async def get_project_metrics(session: AsyncSession, project: ProjectModel) -> str:
@@ -33,20 +186,20 @@ async def get_project_metrics(session: AsyncSession, project: ProjectModel) -> s
             JobModel.status.in_([JobStatus.RUNNING]),
         )
         .order_by(JobModel.job_name)
-        .options(joinedload(JobPrometheusMetrics.job).joinedload(JobModel.project))
+        .options(
+            joinedload(JobPrometheusMetrics.job).joinedload(JobModel.project),
+            joinedload(JobPrometheusMetrics.job)
+            .joinedload(JobModel.run)
+            .joinedload(RunModel.user),
+        )
     )
     metrics_models = res.scalars().all()
-    return _process_metrics(metrics_models)
-
-
-def _process_metrics(metrics_models: Iterable[JobPrometheusMetrics]) -> str:
-    metrics = _parse_and_enrich_metrics(metrics_models)
-    if not metrics:
-        return ""
-    return "\n".join(_render_metrics(metrics)) + "\n"
+    return "\n".join(_render_metrics(_parse_and_enrich_job_gpu_metrics(metrics_models))) + "\n"
 
 
-def _parse_and_enrich_metrics(metrics_models: Iterable[JobPrometheusMetrics]) -> list[Metric]:
+def _parse_and_enrich_job_gpu_metrics(
+    metrics_models: Iterable[JobPrometheusMetrics],
+) -> Iterable[Metric]:
     metrics: dict[str, Metric] = {}
     for metrics_model in metrics_models:
         for metric in text_string_to_metric_families(metrics_model.text):
@@ -56,18 +209,21 @@ def _parse_and_enrich_metrics(metrics_models: Iterable[JobPrometheusMetrics]) ->
             metric = metrics.setdefault(name, metric)
             for sample in samples:
                 labels = sample.labels
-                labels.update(_get_dstack_labels(metrics_model.job))
+                labels.update(_get_job_labels(metrics_model.job))
                 # text_string_to_metric_families "fixes" counter names appending _total,
                 # we rebuild Sample to revert this
                 metric.samples.append(Sample(name, labels, *sample[2:]))
-    return list(metrics.values())
+    return metrics.values()
 
 
-def _get_dstack_labels(job: JobModel) -> dict[str, str]:
+def _get_job_labels(job: JobModel) -> dict[str, str]:
     return {
         "dstack_project_name": job.project.name,
+        "dstack_user_name": job.run.user.name,
         "dstack_run_name": job.run_name,
+        "dstack_run_id": str(job.run_id),
         "dstack_job_name": job.job_name,
+        "dstack_job_id": str(job.id),
         "dstack_job_num": str(job.job_num),
         "dstack_replica_num": str(job.replica_num),
     }
@@ -75,12 +231,14 @@ def _get_dstack_labels(job: JobModel) -> dict[str, str]:
 
 def _render_metrics(metrics: Iterable[Metric]) -> Generator[str, None, None]:
     for metric in metrics:
+        if not metric.samples:
+            continue
         yield f"# HELP {metric.name} {metric.documentation}"
         yield f"# TYPE {metric.name} {metric.type}"
         for sample in metric.samples:
             parts: list[str] = [f"{sample.name}{{"]
             parts.extend(",".join(f'{name}="{value}"' for name, value in sample.labels.items()))
-            parts.append(f"}} {sample.value}")
+            parts.append(f"}} {float(sample.value)}")
             # text_string_to_metric_families converts milliseconds to float seconds
             if isinstance(sample.timestamp, float):
                 parts.append(f" {int(sample.timestamp * 1000)}")

dstack/_internal/server/services/runs.py

@@ -52,7 +52,6 @@ from dstack._internal.server import settings
 from dstack._internal.server.db import get_db
 from dstack._internal.server.models import (
     JobModel,
-    PoolModel,
     ProjectModel,
     RepoModel,
     RunModel,
@@ -61,6 +60,12 @@ from dstack._internal.server.models import (
 from dstack._internal.server.services import repos as repos_services
 from dstack._internal.server.services import services
 from dstack._internal.server.services.docker import is_valid_docker_volume_target
+from dstack._internal.server.services.instances import (
+    filter_pool_instances,
+    get_instance_offer,
+    get_pool_instances,
+    get_shared_pool_instances_with_offers,
+)
 from dstack._internal.server.services.jobs import (
     check_can_attach_job_volumes,
     delay_job_instance_termination,
@@ -74,13 +79,6 @@ from dstack._internal.server.services.jobs import (
 from dstack._internal.server.services.locking import get_locker, string_to_lock_id
 from dstack._internal.server.services.logging import fmt
 from dstack._internal.server.services.offers import get_offers_by_requirements
-from dstack._internal.server.services.pools import (
-    filter_pool_instances,
-    get_instance_offer,
-    get_or_create_pool_by_name,
-    get_pool_instances,
-    get_shared_pool_instances_with_offers,
-)
 from dstack._internal.server.services.projects import list_project_models, list_user_project_models
 from dstack._internal.server.services.users import get_user_model_by_name
 from dstack._internal.utils.logging import get_logger
@@ -308,12 +306,9 @@ async def get_plan(
         job_num=0,
     )
 
-    pool = await get_or_create_pool_by_name(
-        session=session, project=project, pool_name=profile.pool_name
-    )
     pool_offers = await _get_pool_offers(
         session=session,
-        pool=pool,
+        project=project,
         run_spec=run_spec,
         job=jobs[0],
         volumes=volumes,
@@ -342,8 +337,11 @@ async def get_plan(
         job_offers.extend(offer for _, offer in offers)
         job_offers.sort(key=lambda offer: not offer.availability.is_available())
 
+        job_spec = job.job_spec
+        _remove_job_spec_sensitive_info(job_spec)
+
         job_plan = JobPlan(
-            job_spec=job.job_spec,
+            job_spec=job_spec,
             offers=job_offers[:50],
             total_offers=len(job_offers),
             max_price=max((offer.price for offer in job_offers), default=None),
@@ -619,7 +617,10 @@ async def delete_runs(
 
 
 def run_model_to_run(
-    run_model: RunModel, include_job_submissions: bool = True, return_in_api: bool = False
+    run_model: RunModel,
+    include_job_submissions: bool = True,
+    return_in_api: bool = False,
+    include_sensitive: bool = False,
 ) -> Run:
     jobs: List[Job] = []
     run_jobs = sorted(run_model.jobs, key=lambda j: (j.replica_num, j.job_num, j.submission_num))
@@ -634,6 +635,8 @@ def run_model_to_run(
             for job_model in job_submissions:
                 if job_spec is None:
                     job_spec = JobSpec.__response__.parse_raw(job_model.job_spec_data)
+                    if not include_sensitive:
+                        _remove_job_spec_sensitive_info(job_spec)
                 if include_job_submissions:
                     job_submission = job_model_to_job_submission(job_model)
                     if return_in_api:
@@ -680,7 +683,7 @@ def run_model_to_run(
 
 async def _get_pool_offers(
     session: AsyncSession,
-    pool: PoolModel,
+    project: ProjectModel,
     run_spec: RunSpec,
     job: Job,
     volumes: List[List[Volume]],
@@ -688,7 +691,8 @@ async def _get_pool_offers(
     pool_offers: list[InstanceOfferWithAvailability] = []
 
     detaching_instances_ids = await get_instances_ids_with_detaching_volumes(session)
-    pool_instances = [i for i in get_pool_instances(pool) if i.id not in detaching_instances_ids]
+    pool_instances = await get_pool_instances(session, project)
+    pool_instances = [i for i in pool_instances if i.id not in detaching_instances_ids]
     multinode = job.job_spec.jobs_per_replica > 1
 
     if not multinode:
@@ -1046,3 +1050,7 @@ async def retry_run_replica_jobs(
         # dirty hack to avoid passing all job submissions
         new_job_model.submission_num = job_model.submission_num + 1
         session.add(new_job_model)
+
+
+def _remove_job_spec_sensitive_info(spec: JobSpec):
+    spec.ssh_key = None

dstack/_internal/server/services/volumes.py

@@ -7,6 +7,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import joinedload, selectinload
 
 from dstack._internal.core.backends import BACKENDS_WITH_VOLUMES_SUPPORT
+from dstack._internal.core.backends.base.compute import ComputeWithVolumeSupport
 from dstack._internal.core.errors import (
     BackendNotAvailable,
     ResourceExistsError,
@@ -32,11 +33,11 @@ from dstack._internal.server.models import (
     VolumeModel,
 )
 from dstack._internal.server.services import backends as backends_services
+from dstack._internal.server.services.instances import get_instance_provisioning_data
 from dstack._internal.server.services.locking import (
     get_locker,
     string_to_lock_id,
 )
-from dstack._internal.server.services.pools import get_instance_provisioning_data
 from dstack._internal.server.services.projects import list_project_models, list_user_project_models
 from dstack._internal.utils import common, random_names
 from dstack._internal.utils.logging import get_logger
@@ -375,10 +376,11 @@ async def generate_volume_name(session: AsyncSession, project: ProjectModel) ->
 def _validate_volume_configuration(configuration: VolumeConfiguration):
     if configuration.volume_id is None and configuration.size is None:
         raise ServerClientError("Volume must specify either volume_id or size")
+    backends_services.check_backend_type_available(configuration.backend)
     if configuration.backend not in BACKENDS_WITH_VOLUMES_SUPPORT:
         raise ServerClientError(
-            f"Volumes are not supported for {configuration.backend.value} backend. "
-            f"Supported backends: {[b.value for b in BACKENDS_WITH_VOLUMES_SUPPORT]}."
+            f"Volumes are not supported for {configuration.backend.value} backend."
+            f" Available backends with volumes support: {[b.value for b in BACKENDS_WITH_VOLUMES_SUPPORT]}."
         )
     if configuration.name is not None:
         validate_dstack_resource_name(configuration.name)
@@ -409,7 +411,9 @@ async def _delete_volume(session: AsyncSession, project: ProjectModel, volume_mo
         )
         return
 
+    compute = backend.compute()
+    assert isinstance(compute, ComputeWithVolumeSupport)
     await common.run_async(
-        backend.compute().delete_volume,
+        compute.delete_volume,
         volume=volume,
     )

dstack/_internal/server/statics/index.html

@@ -1,3 +1,3 @@
 <!doctype html><html lang="en"><head><meta charset="utf-8"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><title>dstack</title><meta name="description" content="Get GPUs at the best prices and availability from a wide range of providers. No cloud account of your own is required.
 "/><link rel="preconnect" href="https://fonts.googleapis.com"><link rel="preconnect" href="https://fonts.gstatic.com" crossorigin><link href="https://fonts.googleapis.com/css2?family=Roboto:ital,wght@0,100;0,300;0,400;0,500;0,700;0,900;1,100;1,300;1,400;1,500;1,700;1,900&display=swap" rel="stylesheet"><meta name="og:title" content="dstack"><meta name="og:type" content="article"><meta name="og:image" content="/splash_thumbnail.png"><meta name="og:description" content="Get GPUs at the best prices and availability from a wide range of providers. No cloud account of your own is required.
-"><link rel="icon" type="image/x-icon" href="/assets/favicon.ico"><link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png"><link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png"><link rel="icon" type="image/png" sizes="48x48" href="/assets/favicon-48x48.png"><link rel="manifest" href="/assets/manifest.webmanifest"><meta name="mobile-web-app-capable" content="yes"><meta name="theme-color" content="#fff"><meta name="application-name" content="dstackai"><link rel="apple-touch-icon" sizes="57x57" href="/assets/apple-touch-icon-57x57.png"><link rel="apple-touch-icon" sizes="60x60" href="/assets/apple-touch-icon-60x60.png"><link rel="apple-touch-icon" sizes="72x72" href="/assets/apple-touch-icon-72x72.png"><link rel="apple-touch-icon" sizes="76x76" href="/assets/apple-touch-icon-76x76.png"><link rel="apple-touch-icon" sizes="114x114" href="/assets/apple-touch-icon-114x114.png"><link rel="apple-touch-icon" sizes="120x120" href="/assets/apple-touch-icon-120x120.png"><link rel="apple-touch-icon" sizes="144x144" href="/assets/apple-touch-icon-144x144.png"><link rel="apple-touch-icon" sizes="152x152" href="/assets/apple-touch-icon-152x152.png"><link rel="apple-touch-icon" sizes="167x167" href="/assets/apple-touch-icon-167x167.png"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon-180x180.png"><link rel="apple-touch-icon" sizes="1024x1024" href="/assets/apple-touch-icon-1024x1024.png"><meta name="apple-mobile-web-app-capable" content="yes"><meta name="apple-mobile-web-app-status-bar-style" content="black-translucent"><meta name="apple-mobile-web-app-title" content="dstackai"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-640x1136.png"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1136x640.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-750x1334.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1334x750.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1125x2436.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2436x1125.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1170x2532.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2532x1170.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1179x2556.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2556x1179.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-828x1792.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1792x828.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2688.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2688x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2208.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2208x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1284x2778.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2778x1284.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1290x2796.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2796x1290.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1488x2266.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2266x1488.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1536x2048.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2048x1536.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1620x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1620.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1640x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1640.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2388.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2388x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2224.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2224x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-2048x2732.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2732x2048.png"><meta name="msapplication-TileColor" content="#fff"><meta name="msapplication-TileImage" content="/assets/mstile-144x144.png"><meta name="msapplication-config" content="/assets/browserconfig.xml"><link rel="yandex-tableau-widget" href="/assets/yandex-browser-manifest.json"><script defer="defer" src="/main-4eb116b97819badd1e2c.js"></script><link href="/main-da9f8c06a69c20dac23e.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div class="b-page-header" id="header"></div><div id="root"></div></body></html>
+"><link rel="icon" type="image/x-icon" href="/assets/favicon.ico"><link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png"><link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png"><link rel="icon" type="image/png" sizes="48x48" href="/assets/favicon-48x48.png"><link rel="manifest" href="/assets/manifest.webmanifest"><meta name="mobile-web-app-capable" content="yes"><meta name="theme-color" content="#fff"><meta name="application-name" content="dstackai"><link rel="apple-touch-icon" sizes="57x57" href="/assets/apple-touch-icon-57x57.png"><link rel="apple-touch-icon" sizes="60x60" href="/assets/apple-touch-icon-60x60.png"><link rel="apple-touch-icon" sizes="72x72" href="/assets/apple-touch-icon-72x72.png"><link rel="apple-touch-icon" sizes="76x76" href="/assets/apple-touch-icon-76x76.png"><link rel="apple-touch-icon" sizes="114x114" href="/assets/apple-touch-icon-114x114.png"><link rel="apple-touch-icon" sizes="120x120" href="/assets/apple-touch-icon-120x120.png"><link rel="apple-touch-icon" sizes="144x144" href="/assets/apple-touch-icon-144x144.png"><link rel="apple-touch-icon" sizes="152x152" href="/assets/apple-touch-icon-152x152.png"><link rel="apple-touch-icon" sizes="167x167" href="/assets/apple-touch-icon-167x167.png"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon-180x180.png"><link rel="apple-touch-icon" sizes="1024x1024" href="/assets/apple-touch-icon-1024x1024.png"><meta name="apple-mobile-web-app-capable" content="yes"><meta name="apple-mobile-web-app-status-bar-style" content="black-translucent"><meta name="apple-mobile-web-app-title" content="dstackai"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-640x1136.png"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1136x640.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-750x1334.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1334x750.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1125x2436.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2436x1125.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1170x2532.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2532x1170.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1179x2556.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2556x1179.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-828x1792.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1792x828.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2688.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2688x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2208.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2208x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1284x2778.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2778x1284.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1290x2796.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2796x1290.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1488x2266.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2266x1488.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1536x2048.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2048x1536.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1620x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1620.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1640x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1640.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2388.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2388x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2224.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2224x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-2048x2732.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2732x2048.png"><meta name="msapplication-TileColor" content="#fff"><meta name="msapplication-TileImage" content="/assets/mstile-144x144.png"><meta name="msapplication-config" content="/assets/browserconfig.xml"><link rel="yandex-tableau-widget" href="/assets/yandex-browser-manifest.json"><script defer="defer" src="/main-4fd5a4770eff59325ee3.js"></script><link href="/main-da9f8c06a69c20dac23e.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div class="b-page-header" id="header"></div><div id="root"></div></body></html>