dstack 0.19.0rc1__py3-none-any.whl → 0.19.2__py3-none-any.whl

dstack/_internal/cli/commands/metrics.py

@@ -0,0 +1,138 @@
+import argparse
+import time
+from typing import Any, Dict, List, Optional, Union
+
+from rich.live import Live
+from rich.table import Table
+
+from dstack._internal.cli.commands import APIBaseCommand
+from dstack._internal.cli.services.completion import RunNameCompleter
+from dstack._internal.cli.utils.common import (
+    LIVE_TABLE_PROVISION_INTERVAL_SECS,
+    LIVE_TABLE_REFRESH_RATE_PER_SEC,
+    add_row_from_dict,
+    console,
+)
+from dstack._internal.core.errors import CLIError
+from dstack._internal.core.models.instances import Resources
+from dstack._internal.core.models.metrics import JobMetrics
+from dstack.api._public import Client
+from dstack.api._public.runs import Run
+
+
+class MetricsCommand(APIBaseCommand):
+    NAME = "metrics"
+    DESCRIPTION = "Show run metrics"
+
+    def _register(self):
+        super()._register()
+        self._parser.add_argument("run_name").completer = RunNameCompleter()
+        self._parser.add_argument(
+            "-w",
+            "--watch",
+            help="Watch run metrics in realtime",
+            action="store_true",
+        )
+
+    def _command(self, args: argparse.Namespace):
+        super()._command(args)
+        run = self.api.runs.get(run_name=args.run_name)
+        if run is None:
+            raise CLIError(f"Run {args.run_name} not found")
+        if run.status.is_finished():
+            raise CLIError(f"Run {args.run_name} is finished")
+        metrics = _get_run_jobs_metrics(api=self.api, run=run)
+
+        if not args.watch:
+            console.print(_get_metrics_table(run, metrics))
+            return
+
+        try:
+            with Live(console=console, refresh_per_second=LIVE_TABLE_REFRESH_RATE_PER_SEC) as live:
+                while True:
+                    live.update(_get_metrics_table(run, metrics))
+                    time.sleep(LIVE_TABLE_PROVISION_INTERVAL_SECS)
+                    run = self.api.runs.get(run_name=args.run_name)
+                    if run is None:
+                        raise CLIError(f"Run {args.run_name} not found")
+                    if run.status.is_finished():
+                        raise CLIError(f"Run {args.run_name} is finished")
+                    metrics = _get_run_jobs_metrics(api=self.api, run=run)
+        except KeyboardInterrupt:
+            pass
+
+
+def _get_run_jobs_metrics(api: Client, run: Run) -> List[JobMetrics]:
+    metrics = []
+    for job in run._run.jobs:
+        job_metrics = api.client.metrics.get_job_metrics(
+            project_name=api.project,
+            run_name=run.name,
+            replica_num=job.job_spec.replica_num,
+            job_num=job.job_spec.job_num,
+        )
+        metrics.append(job_metrics)
+    return metrics
+
+
+def _get_metrics_table(run: Run, metrics: List[JobMetrics]) -> Table:
+    table = Table(box=None)
+    table.add_column("NAME", style="bold", no_wrap=True)
+    table.add_column("CPU")
+    table.add_column("MEMORY")
+    table.add_column("GPU")
+
+    run_row: Dict[Union[str, int], Any] = {"NAME": run.name}
+    if len(run._run.jobs) != 1:
+        add_row_from_dict(table, run_row)
+
+    for job, job_metrics in zip(run._run.jobs, metrics):
+        jrd = job.job_submissions[-1].job_runtime_data
+        jpd = job.job_submissions[-1].job_provisioning_data
+        resources: Optional[Resources] = None
+        if jrd is not None and jrd.offer is not None:
+            resources = jrd.offer.instance.resources
+        elif jpd is not None:
+            resources = jpd.instance_type.resources
+        cpu_usage = _get_metric_value(job_metrics, "cpu_usage_percent")
+        if cpu_usage is not None:
+            if resources is not None:
+                cpu_usage = cpu_usage / resources.cpus
+            cpu_usage = f"{cpu_usage:.0f}%"
+        memory_usage = _get_metric_value(job_metrics, "memory_working_set_bytes")
+        if memory_usage is not None:
+            memory_usage = f"{round(memory_usage / 1024 / 1024)}MB"
+            if resources is not None:
+                memory_usage += f"/{resources.memory_mib}MB"
+        gpu_metrics = ""
+        gpus_detected_num = _get_metric_value(job_metrics, "gpus_detected_num")
+        if gpus_detected_num is not None:
+            for i in range(gpus_detected_num):
+                gpu_memory_usage = _get_metric_value(job_metrics, f"gpu_memory_usage_bytes_gpu{i}")
+                gpu_util_percent = _get_metric_value(job_metrics, f"gpu_util_percent_gpu{i}")
+                if gpu_memory_usage is not None:
+                    if i != 0:
+                        gpu_metrics += "\n"
+                    gpu_metrics += f"#{i} {round(gpu_memory_usage / 1024 / 1024)}MB"
+                    if resources is not None:
+                        gpu_metrics += f"/{resources.gpus[i].memory_mib}MB"
+                    gpu_metrics += f" {gpu_util_percent}% Util"
+
+        job_row: Dict[Union[str, int], Any] = {
+            "NAME": f"  replica={job.job_spec.replica_num} job={job.job_spec.job_num}",
+            "CPU": cpu_usage or "-",
+            "MEMORY": memory_usage or "-",
+            "GPU": gpu_metrics or "-",
+        }
+        if len(run._run.jobs) == 1:
+            job_row.update(run_row)
+        add_row_from_dict(table, job_row)
+
+    return table
+
+
+def _get_metric_value(job_metrics: JobMetrics, name: str) -> Optional[Any]:
+    for metric in job_metrics.metrics:
+        if metric.name == name:
+            return metric.values[-1]
+    return None

dstack/_internal/cli/commands/stats.py

@@ -1,128 +1,14 @@
 import argparse
-import time
-from typing import Any, Dict, List, Optional, Union
 
-from rich.live import Live
-from rich.table import Table
+from dstack._internal.cli.commands.metrics import MetricsCommand
+from dstack._internal.utils.logging import get_logger
 
-from dstack._internal.cli.commands import APIBaseCommand
-from dstack._internal.cli.services.completion import RunNameCompleter
-from dstack._internal.cli.utils.common import (
-    LIVE_TABLE_PROVISION_INTERVAL_SECS,
-    LIVE_TABLE_REFRESH_RATE_PER_SEC,
-    add_row_from_dict,
-    console,
-)
-from dstack._internal.core.errors import CLIError
-from dstack._internal.core.models.metrics import JobMetrics
-from dstack.api._public import Client
-from dstack.api._public.runs import Run
+logger = get_logger(__name__)
 
 
-class StatsCommand(APIBaseCommand):
+class StatsCommand(MetricsCommand):
     NAME = "stats"
-    DESCRIPTION = "Show run stats"
-
-    def _register(self):
-        super()._register()
-        self._parser.add_argument("run_name").completer = RunNameCompleter()
-        self._parser.add_argument(
-            "-w",
-            "--watch",
-            help="Watch run stats in realtime",
-            action="store_true",
-        )
 
     def _command(self, args: argparse.Namespace):
+        logger.warning("`dstack stats` is deprecated in favor of `dstack metrics`")
         super()._command(args)
-        run = self.api.runs.get(run_name=args.run_name)
-        if run is None:
-            raise CLIError(f"Run {args.run_name} not found")
-        if run.status.is_finished():
-            raise CLIError(f"Run {args.run_name} is finished")
-        metrics = _get_run_jobs_metrics(api=self.api, run=run)
-
-        if not args.watch:
-            console.print(_get_stats_table(run, metrics))
-            return
-
-        try:
-            with Live(console=console, refresh_per_second=LIVE_TABLE_REFRESH_RATE_PER_SEC) as live:
-                while True:
-                    live.update(_get_stats_table(run, metrics))
-                    time.sleep(LIVE_TABLE_PROVISION_INTERVAL_SECS)
-                    run = self.api.runs.get(run_name=args.run_name)
-                    if run is None:
-                        raise CLIError(f"Run {args.run_name} not found")
-                    if run.status.is_finished():
-                        raise CLIError(f"Run {args.run_name} is finished")
-                    metrics = _get_run_jobs_metrics(api=self.api, run=run)
-        except KeyboardInterrupt:
-            pass
-
-
-def _get_run_jobs_metrics(api: Client, run: Run) -> List[JobMetrics]:
-    metrics = []
-    for job in run._run.jobs:
-        job_metrics = api.client.metrics.get_job_metrics(
-            project_name=api.project,
-            run_name=run.name,
-            replica_num=job.job_spec.replica_num,
-            job_num=job.job_spec.job_num,
-        )
-        metrics.append(job_metrics)
-    return metrics
-
-
-def _get_stats_table(run: Run, metrics: List[JobMetrics]) -> Table:
-    table = Table(box=None)
-    table.add_column("NAME", style="bold", no_wrap=True)
-    table.add_column("CPU")
-    table.add_column("MEMORY")
-    table.add_column("GPU")
-
-    run_row: Dict[Union[str, int], Any] = {"NAME": run.name}
-    if len(run._run.jobs) != 1:
-        add_row_from_dict(table, run_row)
-
-    for job, job_metrics in zip(run._run.jobs, metrics):
-        cpu_usage = _get_metric_value(job_metrics, "cpu_usage_percent")
-        if cpu_usage is not None:
-            cpu_usage = f"{cpu_usage}%"
-        memory_usage = _get_metric_value(job_metrics, "memory_working_set_bytes")
-        if memory_usage is not None:
-            memory_usage = f"{round(memory_usage / 1024 / 1024)}MB"
-            if job.job_submissions[-1].job_provisioning_data is not None:
-                memory_usage += f"/{job.job_submissions[-1].job_provisioning_data.instance_type.resources.memory_mib}MB"
-        gpu_stats = ""
-        gpus_detected_num = _get_metric_value(job_metrics, "gpus_detected_num")
-        if gpus_detected_num is not None:
-            for i in range(gpus_detected_num):
-                gpu_memory_usage = _get_metric_value(job_metrics, f"gpu_memory_usage_bytes_gpu{i}")
-                gpu_util_percent = _get_metric_value(job_metrics, f"gpu_util_percent_gpu{i}")
-                if gpu_memory_usage is not None:
-                    if i != 0:
-                        gpu_stats += "\n"
-                    gpu_stats += f"#{i} {round(gpu_memory_usage / 1024 / 1024)}MB"
-                    if job.job_submissions[-1].job_provisioning_data is not None:
-                        gpu_stats += f"/{job.job_submissions[-1].job_provisioning_data.instance_type.resources.gpus[i].memory_mib}MB"
-                    gpu_stats += f" {gpu_util_percent}% Util"
-
-        job_row: Dict[Union[str, int], Any] = {
-            "NAME": f"  replica={job.job_spec.replica_num} job={job.job_spec.job_num}",
-            "CPU": cpu_usage or "-",
-            "MEMORY": memory_usage or "-",
-            "GPU": gpu_stats or "-",
-        }
-        if len(run._run.jobs) == 1:
-            job_row.update(run_row)
-        add_row_from_dict(table, job_row)
-
-    return table
-
-
-def _get_metric_value(job_metrics: JobMetrics, name: str) -> Optional[Any]:
-    for metric in job_metrics.metrics:
-        if metric.name == name:
-            return metric.values[-1]
-    return None

dstack/_internal/cli/main.py

@@ -13,6 +13,7 @@ from dstack._internal.cli.commands.fleet import FleetCommand
 from dstack._internal.cli.commands.gateway import GatewayCommand
 from dstack._internal.cli.commands.init import InitCommand
 from dstack._internal.cli.commands.logs import LogsCommand
+from dstack._internal.cli.commands.metrics import MetricsCommand
 from dstack._internal.cli.commands.ps import PsCommand
 from dstack._internal.cli.commands.server import ServerCommand
 from dstack._internal.cli.commands.stats import StatsCommand
@@ -65,6 +66,7 @@ def main():
     GatewayCommand.register(subparsers)
     InitCommand.register(subparsers)
     LogsCommand.register(subparsers)
+    MetricsCommand.register(subparsers)
     PsCommand.register(subparsers)
     ServerCommand.register(subparsers)
     StatsCommand.register(subparsers)

dstack/_internal/core/backends/base/compute.py

@@ -94,6 +94,9 @@ class Compute(ABC):
         """
         Terminates an instance by `instance_id`.
         If the instance does not exist, it should not raise errors but return silently.
+
+        Should return ASAP. If required to wait for some operation, raise `NotYetTerminated`.
+        In this case, the method will be called again after a few seconds.
         """
         pass
 

dstack/_internal/core/backends/base/models.py

@@ -1,14 +1,14 @@
 from pathlib import Path
 
 
-def fill_data(values: dict):
-    if values.get("data") is not None:
+def fill_data(values: dict, filename_field: str = "filename", data_field: str = "data") -> dict:
+    if values.get(data_field) is not None:
         return values
-    if "filename" not in values:
-        raise ValueError()
+    if (filename := values.get(filename_field)) is None:
+        raise ValueError(f"Either `{filename_field}` or `{data_field}` must be specified")
     try:
-        with open(Path(values["filename"]).expanduser()) as f:
-            values["data"] = f.read()
+        with open(Path(filename).expanduser()) as f:
+            values[data_field] = f.read()
     except OSError:
-        raise ValueError(f"No such file {values['filename']}")
+        raise ValueError(f"No such file {filename}")
     return values

dstack/_internal/core/backends/configurators.py

@@ -63,6 +63,15 @@ try:
 except ImportError:
     pass
 
+try:
+    from dstack._internal.core.backends.nebius.configurator import (
+        NebiusConfigurator,
+    )
+
+    _CONFIGURATOR_CLASSES.append(NebiusConfigurator)
+except ImportError:
+    pass
+
 try:
     from dstack._internal.core.backends.oci.configurator import OCIConfigurator
 

dstack/_internal/core/backends/models.py

@@ -34,6 +34,11 @@ from dstack._internal.core.backends.lambdalabs.models import (
     LambdaBackendConfig,
     LambdaBackendConfigWithCreds,
 )
+from dstack._internal.core.backends.nebius.models import (
+    NebiusBackendConfig,
+    NebiusBackendConfigWithCreds,
+    NebiusBackendFileConfigWithCreds,
+)
 from dstack._internal.core.backends.oci.models import (
     OCIBackendConfig,
     OCIBackendConfigWithCreds,
@@ -65,6 +70,7 @@ AnyBackendConfigWithoutCreds = Union[
     GCPBackendConfig,
     KubernetesBackendConfig,
     LambdaBackendConfig,
+    NebiusBackendConfig,
     OCIBackendConfig,
     RunpodBackendConfig,
     TensorDockBackendConfig,
@@ -86,6 +92,7 @@ AnyBackendConfigWithCreds = Union[
     KubernetesBackendConfigWithCreds,
     LambdaBackendConfigWithCreds,
     OCIBackendConfigWithCreds,
+    NebiusBackendConfigWithCreds,
     RunpodBackendConfigWithCreds,
     TensorDockBackendConfigWithCreds,
     VastAIBackendConfigWithCreds,
@@ -105,6 +112,7 @@ AnyBackendFileConfigWithCreds = Union[
     KubernetesBackendFileConfigWithCreds,
     LambdaBackendConfigWithCreds,
     OCIBackendConfigWithCreds,
+    NebiusBackendFileConfigWithCreds,
     RunpodBackendConfigWithCreds,
     TensorDockBackendConfigWithCreds,
     VastAIBackendConfigWithCreds,

dstack/_internal/core/backends/nebius/backend.py

@@ -0,0 +1,16 @@
+from dstack._internal.core.backends.base.backend import Backend
+from dstack._internal.core.backends.nebius.compute import NebiusCompute
+from dstack._internal.core.backends.nebius.models import NebiusConfig
+from dstack._internal.core.models.backends.base import BackendType
+
+
+class NebiusBackend(Backend):
+    TYPE = BackendType.NEBIUS
+    COMPUTE_CLASS = NebiusCompute
+
+    def __init__(self, config: NebiusConfig):
+        self.config = config
+        self._compute = NebiusCompute(self.config)
+
+    def compute(self) -> NebiusCompute:
+        return self._compute

dstack/_internal/core/backends/nebius/compute.py

@@ -0,0 +1,270 @@
+import json
+import shlex
+import time
+from functools import cached_property
+from typing import List, Optional
+
+from nebius.aio.operation import Operation as SDKOperation
+from nebius.aio.service_error import RequestError, StatusCode
+from nebius.api.nebius.common.v1 import Operation
+from nebius.sdk import SDK
+
+from dstack._internal.core.backends.base.backend import Compute
+from dstack._internal.core.backends.base.compute import (
+    ComputeWithCreateInstanceSupport,
+    ComputeWithMultinodeSupport,
+    generate_unique_instance_name,
+    get_user_data,
+)
+from dstack._internal.core.backends.base.offers import get_catalog_offers
+from dstack._internal.core.backends.nebius import resources
+from dstack._internal.core.backends.nebius.models import NebiusConfig, NebiusServiceAccountCreds
+from dstack._internal.core.errors import BackendError, NotYetTerminated, ProvisioningError
+from dstack._internal.core.models.backends.base import BackendType
+from dstack._internal.core.models.common import CoreModel
+from dstack._internal.core.models.instances import (
+    InstanceAvailability,
+    InstanceConfiguration,
+    InstanceOffer,
+    InstanceOfferWithAvailability,
+)
+from dstack._internal.core.models.resources import Memory, Range
+from dstack._internal.core.models.runs import JobProvisioningData, Requirements
+from dstack._internal.utils.logging import get_logger
+
+logger = get_logger(__name__)
+CONFIGURABLE_DISK_SIZE = Range[Memory](
+    min=Memory.parse("40GB"),  # min for the ubuntu22.04-cuda12 image
+    max=Memory.parse("8192GB"),  # max for the NETWORK_SSD disk type
+)
+WAIT_FOR_DISK_TIMEOUT = 20
+WAIT_FOR_INSTANCE_TIMEOUT = 30
+WAIT_FOR_INSTANCE_UPDATE_INTERVAL = 2.5
+DELETE_INSTANCE_TIMEOUT = 25
+DOCKER_DAEMON_CONFIG = {
+    "runtimes": {"nvidia": {"args": [], "path": "nvidia-container-runtime"}},
+    # Workaround for https://github.com/NVIDIA/nvidia-container-toolkit/issues/48
+    "exec-opts": ["native.cgroupdriver=cgroupfs"],
+}
+SETUP_COMMANDS = [
+    "ufw allow ssh",
+    "ufw allow from 192.168.0.0/16",
+    "ufw default deny incoming",
+    "ufw default allow outgoing",
+    "ufw enable",
+    'sed -i "s/.*AllowTcpForwarding.*/AllowTcpForwarding yes/g" /etc/ssh/sshd_config',
+    "service ssh restart",
+    f"echo {shlex.quote(json.dumps(DOCKER_DAEMON_CONFIG))} > /etc/docker/daemon.json",
+    "service docker restart",
+]
+SUPPORTED_PLATFORMS = [
+    "gpu-h100-sxm",
+    "gpu-h200-sxm",
+    "gpu-l40s-a",
+    "gpu-l40s-d",
+    "cpu-d3",
+    "cpu-e2",
+]
+
+
+class NebiusCompute(
+    ComputeWithCreateInstanceSupport,
+    ComputeWithMultinodeSupport,
+    Compute,
+):
+    def __init__(self, config: NebiusConfig):
+        super().__init__()
+        self.config = config
+        self._subnet_id_cache: dict[str, str] = {}
+
+    @cached_property
+    def _sdk(self) -> SDK:
+        assert isinstance(self.config.creds, NebiusServiceAccountCreds)
+        return resources.make_sdk(self.config.creds)
+
+    @cached_property
+    def _region_to_project_id(self) -> dict[str, str]:
+        return resources.get_region_to_project_id_map(self._sdk)
+
+    def _get_subnet_id(self, region: str) -> str:
+        if region not in self._subnet_id_cache:
+            self._subnet_id_cache[region] = resources.get_default_subnet(
+                self._sdk, self._region_to_project_id[region]
+            ).metadata.id
+        return self._subnet_id_cache[region]
+
+    def get_offers(
+        self, requirements: Optional[Requirements] = None
+    ) -> List[InstanceOfferWithAvailability]:
+        offers = get_catalog_offers(
+            backend=BackendType.NEBIUS,
+            locations=self.config.regions or list(self._region_to_project_id),
+            requirements=requirements,
+            extra_filter=_supported_instances,
+            configurable_disk_size=CONFIGURABLE_DISK_SIZE,
+        )
+        return [
+            InstanceOfferWithAvailability(
+                **offer.dict(),
+                availability=InstanceAvailability.UNKNOWN,
+            )
+            for offer in offers
+        ]
+
+    def create_instance(
+        self,
+        instance_offer: InstanceOfferWithAvailability,
+        instance_config: InstanceConfiguration,
+    ) -> JobProvisioningData:
+        # NOTE: This method can block for a long time as it waits for the boot disk to be created
+        # and the instance to enter the STARTING state. This has to be done in create_instance so
+        # that we can handle quota and availability errors that may occur even after creating an
+        # instance.
+        instance_name = generate_unique_instance_name(instance_config)
+        platform, preset = instance_offer.instance.name.split()
+        create_disk_op = resources.create_disk(
+            sdk=self._sdk,
+            name=instance_name,
+            project_id=self._region_to_project_id[instance_offer.region],
+            size_mib=instance_offer.instance.resources.disk.size_mib,
+            image_family="ubuntu22.04-cuda12",
+        )
+        create_instance_op = None
+        try:
+            logger.debug("Blocking until disk %s is created", create_disk_op.resource_id)
+            resources.wait_for_operation(create_disk_op, timeout=WAIT_FOR_DISK_TIMEOUT)
+            if not create_disk_op.successful():
+                raw_op = create_disk_op.raw()
+                raise ProvisioningError(
+                    f"Create disk operation failed. Message: {raw_op.status.message}."
+                    f" Details: {raw_op.status.details}"
+                )
+            create_instance_op = resources.create_instance(
+                sdk=self._sdk,
+                name=instance_name,
+                project_id=self._region_to_project_id[instance_offer.region],
+                user_data=get_user_data(
+                    instance_config.get_public_keys(),
+                    backend_specific_commands=SETUP_COMMANDS,
+                ),
+                platform=platform,
+                preset=preset,
+                disk_id=create_disk_op.resource_id,
+                subnet_id=self._get_subnet_id(instance_offer.region),
+            )
+            _wait_for_instance(self._sdk, create_instance_op)
+        except BaseException:
+            if create_instance_op is not None:
+                try:
+                    with resources.ignore_errors([StatusCode.NOT_FOUND]):
+                        delete_instance_op = resources.delete_instance(
+                            self._sdk, create_instance_op.resource_id
+                        )
+                    resources.wait_for_operation(
+                        delete_instance_op, timeout=DELETE_INSTANCE_TIMEOUT
+                    )
+                except Exception as e:
+                    logger.exception(
+                        "Could not delete instance %s: %s", create_instance_op.resource_id, e
+                    )
+            try:
+                with resources.ignore_errors([StatusCode.NOT_FOUND]):
+                    resources.delete_disk(self._sdk, create_disk_op.resource_id)
+            except Exception as e:
+                logger.exception(
+                    "Could not delete boot disk %s: %s", create_disk_op.resource_id, e
+                )
+            raise
+        return JobProvisioningData(
+            backend=instance_offer.backend,
+            instance_type=instance_offer.instance,
+            instance_id=create_instance_op.resource_id,
+            hostname=None,
+            region=instance_offer.region,
+            price=instance_offer.price,
+            ssh_port=22,
+            username="ubuntu",
+            dockerized=True,
+            backend_data=NebiusInstanceBackendData(boot_disk_id=create_disk_op.resource_id).json(),
+        )
+
+    def update_provisioning_data(
+        self, provisioning_data, project_ssh_public_key, project_ssh_private_key
+    ):
+        instance = resources.get_instance(self._sdk, provisioning_data.instance_id)
+        if not instance.status.network_interfaces:
+            return
+        interface = instance.status.network_interfaces[0]
+        provisioning_data.hostname, _ = interface.public_ip_address.address.split("/")
+        provisioning_data.internal_ip, _ = interface.ip_address.address.split("/")
+
+    def terminate_instance(
+        self, instance_id: str, region: str, backend_data: Optional[str] = None
+    ):
+        backend_data_parsed = NebiusInstanceBackendData.load(backend_data)
+        try:
+            instance = resources.get_instance(self._sdk, instance_id)
+        except RequestError as e:
+            if e.status.code != StatusCode.NOT_FOUND:
+                raise
+            instance = None
+        if instance is not None:
+            if instance.status.state != instance.status.InstanceState.DELETING:
+                resources.delete_instance(self._sdk, instance_id)
+                raise NotYetTerminated(
+                    "Requested instance deletion."
+                    " Will wait for deletion before deleting the boot disk."
+                    f" Instance state was: {instance.status.state.name}"
+                )
+            else:
+                raise NotYetTerminated(
+                    "Waiting for instance deletion before deleting the boot disk."
+                    f" Instance state: {instance.status.state.name}"
+                )
+        with resources.ignore_errors([StatusCode.NOT_FOUND]):
+            resources.delete_disk(self._sdk, backend_data_parsed.boot_disk_id)
+
+
+class NebiusInstanceBackendData(CoreModel):
+    boot_disk_id: str
+
+    @classmethod
+    def load(cls, raw: Optional[str]) -> "NebiusInstanceBackendData":
+        assert raw is not None
+        return cls.__response__.parse_raw(raw)
+
+
+def _wait_for_instance(sdk: SDK, op: SDKOperation[Operation]) -> None:
+    start = time.monotonic()
+    while True:
+        if op.done() and not op.successful():
+            raise ProvisioningError(
+                f"Create instance operation failed. Message: {op.raw().status.message}."
+                f" Details: {op.raw().status.details}"
+            )
+        instance = resources.get_instance(sdk, op.resource_id)
+        if instance.status.state in [
+            instance.status.InstanceState.STARTING,
+            instance.status.InstanceState.RUNNING,
+        ]:
+            break
+        if time.monotonic() - start > WAIT_FOR_INSTANCE_TIMEOUT:
+            raise BackendError(
+                f"Instance {instance.metadata.id} did not start booting in time."
+                f" Status: {instance.status.state.name}"
+            )
+        logger.debug(
+            "Waiting for instance %s. Status: %s. Operation status: %s",
+            instance.metadata.name,
+            instance.status.state.name,
+            op.status(),
+        )
+        time.sleep(WAIT_FOR_INSTANCE_UPDATE_INTERVAL)
+        resources.LOOP.await_(
+            op.update(timeout=resources.REQUEST_TIMEOUT, metadata=resources.REQUEST_MD)
+        )
+
+
+def _supported_instances(offer: InstanceOffer) -> bool:
+    platform, _ = offer.instance.name.split()
+    return platform in SUPPORTED_PLATFORMS and not offer.instance.resources.spot