dstack 0.19.1__py3-none-any.whl → 0.19.3__py3-none-any.whl

dstack/_internal/core/backends/nebius/compute.py

@@ -0,0 +1,272 @@
+import json
+import shlex
+import time
+from functools import cached_property
+from typing import List, Optional
+
+from nebius.aio.operation import Operation as SDKOperation
+from nebius.aio.service_error import RequestError, StatusCode
+from nebius.api.nebius.common.v1 import Operation
+from nebius.sdk import SDK
+
+from dstack._internal.core.backends.base.backend import Compute
+from dstack._internal.core.backends.base.compute import (
+    ComputeWithCreateInstanceSupport,
+    ComputeWithMultinodeSupport,
+    generate_unique_instance_name,
+    get_user_data,
+)
+from dstack._internal.core.backends.base.offers import get_catalog_offers
+from dstack._internal.core.backends.nebius import resources
+from dstack._internal.core.backends.nebius.models import NebiusConfig, NebiusServiceAccountCreds
+from dstack._internal.core.errors import BackendError, NotYetTerminated, ProvisioningError
+from dstack._internal.core.models.backends.base import BackendType
+from dstack._internal.core.models.common import CoreModel
+from dstack._internal.core.models.instances import (
+    InstanceAvailability,
+    InstanceConfiguration,
+    InstanceOffer,
+    InstanceOfferWithAvailability,
+)
+from dstack._internal.core.models.resources import Memory, Range
+from dstack._internal.core.models.runs import JobProvisioningData, Requirements
+from dstack._internal.utils.logging import get_logger
+
+logger = get_logger(__name__)
+CONFIGURABLE_DISK_SIZE = Range[Memory](
+    min=Memory.parse("40GB"),  # min for the ubuntu22.04-cuda12 image
+    max=Memory.parse("8192GB"),  # max for the NETWORK_SSD disk type
+)
+WAIT_FOR_DISK_TIMEOUT = 20
+WAIT_FOR_INSTANCE_TIMEOUT = 30
+WAIT_FOR_INSTANCE_UPDATE_INTERVAL = 2.5
+DELETE_INSTANCE_TIMEOUT = 25
+DOCKER_DAEMON_CONFIG = {
+    "runtimes": {"nvidia": {"args": [], "path": "nvidia-container-runtime"}},
+    # Workaround for https://github.com/NVIDIA/nvidia-container-toolkit/issues/48
+    "exec-opts": ["native.cgroupdriver=cgroupfs"],
+}
+SETUP_COMMANDS = [
+    "ufw allow ssh",
+    "ufw allow from 10.0.0.0/8",
+    "ufw allow from 172.16.0.0/12",
+    "ufw allow from 192.168.0.0/16",
+    "ufw default deny incoming",
+    "ufw default allow outgoing",
+    "ufw enable",
+    'sed -i "s/.*AllowTcpForwarding.*/AllowTcpForwarding yes/g" /etc/ssh/sshd_config',
+    "service ssh restart",
+    f"echo {shlex.quote(json.dumps(DOCKER_DAEMON_CONFIG))} > /etc/docker/daemon.json",
+    "service docker restart",
+]
+SUPPORTED_PLATFORMS = [
+    "gpu-h100-sxm",
+    "gpu-h200-sxm",
+    "gpu-l40s-a",
+    "gpu-l40s-d",
+    "cpu-d3",
+    "cpu-e2",
+]
+
+
+class NebiusCompute(
+    ComputeWithCreateInstanceSupport,
+    ComputeWithMultinodeSupport,
+    Compute,
+):
+    def __init__(self, config: NebiusConfig):
+        super().__init__()
+        self.config = config
+        self._subnet_id_cache: dict[str, str] = {}
+
+    @cached_property
+    def _sdk(self) -> SDK:
+        assert isinstance(self.config.creds, NebiusServiceAccountCreds)
+        return resources.make_sdk(self.config.creds)
+
+    @cached_property
+    def _region_to_project_id(self) -> dict[str, str]:
+        return resources.get_region_to_project_id_map(self._sdk)
+
+    def _get_subnet_id(self, region: str) -> str:
+        if region not in self._subnet_id_cache:
+            self._subnet_id_cache[region] = resources.get_default_subnet(
+                self._sdk, self._region_to_project_id[region]
+            ).metadata.id
+        return self._subnet_id_cache[region]
+
+    def get_offers(
+        self, requirements: Optional[Requirements] = None
+    ) -> List[InstanceOfferWithAvailability]:
+        offers = get_catalog_offers(
+            backend=BackendType.NEBIUS,
+            locations=self.config.regions or list(self._region_to_project_id),
+            requirements=requirements,
+            extra_filter=_supported_instances,
+            configurable_disk_size=CONFIGURABLE_DISK_SIZE,
+        )
+        return [
+            InstanceOfferWithAvailability(
+                **offer.dict(),
+                availability=InstanceAvailability.UNKNOWN,
+            )
+            for offer in offers
+        ]
+
+    def create_instance(
+        self,
+        instance_offer: InstanceOfferWithAvailability,
+        instance_config: InstanceConfiguration,
+    ) -> JobProvisioningData:
+        # NOTE: This method can block for a long time as it waits for the boot disk to be created
+        # and the instance to enter the STARTING state. This has to be done in create_instance so
+        # that we can handle quota and availability errors that may occur even after creating an
+        # instance.
+        instance_name = generate_unique_instance_name(instance_config)
+        platform, preset = instance_offer.instance.name.split()
+        create_disk_op = resources.create_disk(
+            sdk=self._sdk,
+            name=instance_name,
+            project_id=self._region_to_project_id[instance_offer.region],
+            size_mib=instance_offer.instance.resources.disk.size_mib,
+            image_family="ubuntu22.04-cuda12",
+        )
+        create_instance_op = None
+        try:
+            logger.debug("Blocking until disk %s is created", create_disk_op.resource_id)
+            resources.wait_for_operation(create_disk_op, timeout=WAIT_FOR_DISK_TIMEOUT)
+            if not create_disk_op.successful():
+                raw_op = create_disk_op.raw()
+                raise ProvisioningError(
+                    f"Create disk operation failed. Message: {raw_op.status.message}."
+                    f" Details: {raw_op.status.details}"
+                )
+            create_instance_op = resources.create_instance(
+                sdk=self._sdk,
+                name=instance_name,
+                project_id=self._region_to_project_id[instance_offer.region],
+                user_data=get_user_data(
+                    instance_config.get_public_keys(),
+                    backend_specific_commands=SETUP_COMMANDS,
+                ),
+                platform=platform,
+                preset=preset,
+                disk_id=create_disk_op.resource_id,
+                subnet_id=self._get_subnet_id(instance_offer.region),
+            )
+            _wait_for_instance(self._sdk, create_instance_op)
+        except BaseException:
+            if create_instance_op is not None:
+                try:
+                    with resources.ignore_errors([StatusCode.NOT_FOUND]):
+                        delete_instance_op = resources.delete_instance(
+                            self._sdk, create_instance_op.resource_id
+                        )
+                    resources.wait_for_operation(
+                        delete_instance_op, timeout=DELETE_INSTANCE_TIMEOUT
+                    )
+                except Exception as e:
+                    logger.exception(
+                        "Could not delete instance %s: %s", create_instance_op.resource_id, e
+                    )
+            try:
+                with resources.ignore_errors([StatusCode.NOT_FOUND]):
+                    resources.delete_disk(self._sdk, create_disk_op.resource_id)
+            except Exception as e:
+                logger.exception(
+                    "Could not delete boot disk %s: %s", create_disk_op.resource_id, e
+                )
+            raise
+        return JobProvisioningData(
+            backend=instance_offer.backend,
+            instance_type=instance_offer.instance,
+            instance_id=create_instance_op.resource_id,
+            hostname=None,
+            region=instance_offer.region,
+            price=instance_offer.price,
+            ssh_port=22,
+            username="ubuntu",
+            dockerized=True,
+            backend_data=NebiusInstanceBackendData(boot_disk_id=create_disk_op.resource_id).json(),
+        )
+
+    def update_provisioning_data(
+        self, provisioning_data, project_ssh_public_key, project_ssh_private_key
+    ):
+        instance = resources.get_instance(self._sdk, provisioning_data.instance_id)
+        if not instance.status.network_interfaces:
+            return
+        interface = instance.status.network_interfaces[0]
+        provisioning_data.hostname, _ = interface.public_ip_address.address.split("/")
+        provisioning_data.internal_ip, _ = interface.ip_address.address.split("/")
+
+    def terminate_instance(
+        self, instance_id: str, region: str, backend_data: Optional[str] = None
+    ):
+        backend_data_parsed = NebiusInstanceBackendData.load(backend_data)
+        try:
+            instance = resources.get_instance(self._sdk, instance_id)
+        except RequestError as e:
+            if e.status.code != StatusCode.NOT_FOUND:
+                raise
+            instance = None
+        if instance is not None:
+            if instance.status.state != instance.status.InstanceState.DELETING:
+                resources.delete_instance(self._sdk, instance_id)
+                raise NotYetTerminated(
+                    "Requested instance deletion."
+                    " Will wait for deletion before deleting the boot disk."
+                    f" Instance state was: {instance.status.state.name}"
+                )
+            else:
+                raise NotYetTerminated(
+                    "Waiting for instance deletion before deleting the boot disk."
+                    f" Instance state: {instance.status.state.name}"
+                )
+        with resources.ignore_errors([StatusCode.NOT_FOUND]):
+            resources.delete_disk(self._sdk, backend_data_parsed.boot_disk_id)
+
+
+class NebiusInstanceBackendData(CoreModel):
+    boot_disk_id: str
+
+    @classmethod
+    def load(cls, raw: Optional[str]) -> "NebiusInstanceBackendData":
+        assert raw is not None
+        return cls.__response__.parse_raw(raw)
+
+
+def _wait_for_instance(sdk: SDK, op: SDKOperation[Operation]) -> None:
+    start = time.monotonic()
+    while True:
+        if op.done() and not op.successful():
+            raise ProvisioningError(
+                f"Create instance operation failed. Message: {op.raw().status.message}."
+                f" Details: {op.raw().status.details}"
+            )
+        instance = resources.get_instance(sdk, op.resource_id)
+        if instance.status.state in [
+            instance.status.InstanceState.STARTING,
+            instance.status.InstanceState.RUNNING,
+        ]:
+            break
+        if time.monotonic() - start > WAIT_FOR_INSTANCE_TIMEOUT:
+            raise BackendError(
+                f"Instance {instance.metadata.id} did not start booting in time."
+                f" Status: {instance.status.state.name}"
+            )
+        logger.debug(
+            "Waiting for instance %s. Status: %s. Operation status: %s",
+            instance.metadata.name,
+            instance.status.state.name,
+            op.status(),
+        )
+        time.sleep(WAIT_FOR_INSTANCE_UPDATE_INTERVAL)
+        resources.LOOP.await_(
+            op.update(timeout=resources.REQUEST_TIMEOUT, metadata=resources.REQUEST_MD)
+        )
+
+
+def _supported_instances(offer: InstanceOffer) -> bool:
+    platform, _ = offer.instance.name.split()
+    return platform in SUPPORTED_PLATFORMS and not offer.instance.resources.spot

dstack/_internal/core/backends/nebius/configurator.py

@@ -0,0 +1,74 @@
+import json
+
+from nebius.aio.service_error import RequestError
+
+from dstack._internal.core.backends.base.configurator import (
+    BackendRecord,
+    Configurator,
+    raise_invalid_credentials_error,
+)
+from dstack._internal.core.backends.nebius import resources
+from dstack._internal.core.backends.nebius.backend import NebiusBackend
+from dstack._internal.core.backends.nebius.models import (
+    AnyNebiusBackendConfig,
+    NebiusBackendConfig,
+    NebiusBackendConfigWithCreds,
+    NebiusConfig,
+    NebiusCreds,
+    NebiusServiceAccountCreds,
+    NebiusStoredConfig,
+)
+from dstack._internal.core.models.backends.base import BackendType
+
+
+class NebiusConfigurator(Configurator):
+    TYPE = BackendType.NEBIUS
+    BACKEND_CLASS = NebiusBackend
+
+    def validate_config(self, config: NebiusBackendConfigWithCreds, default_creds_enabled: bool):
+        assert isinstance(config.creds, NebiusServiceAccountCreds)
+        try:
+            sdk = resources.make_sdk(config.creds)
+            available_regions = set(resources.get_region_to_project_id_map(sdk))
+        except (ValueError, RequestError) as e:
+            raise_invalid_credentials_error(
+                fields=[["creds"]],
+                details=str(e),
+            )
+        if invalid_regions := set(config.regions or []) - available_regions:
+            raise_invalid_credentials_error(
+                fields=[["regions"]],
+                details=(
+                    f"Configured regions {invalid_regions} do not exist in this Nebius tenancy."
+                    " Omit `regions` to use all regions or select some of the available regions:"
+                    f" {available_regions}"
+                ),
+            )
+
+    def create_backend(
+        self, project_name: str, config: NebiusBackendConfigWithCreds
+    ) -> BackendRecord:
+        return BackendRecord(
+            config=NebiusStoredConfig(
+                **NebiusBackendConfig.__response__.parse_obj(config).dict()
+            ).json(),
+            auth=NebiusCreds.parse_obj(config.creds).json(),
+        )
+
+    def get_backend_config(
+        self, record: BackendRecord, include_creds: bool
+    ) -> AnyNebiusBackendConfig:
+        config = self._get_config(record)
+        if include_creds:
+            return NebiusBackendConfigWithCreds.__response__.parse_obj(config)
+        return NebiusBackendConfig.__response__.parse_obj(config)
+
+    def get_backend(self, record: BackendRecord) -> NebiusBackend:
+        config = self._get_config(record)
+        return NebiusBackend(config=config)
+
+    def _get_config(self, record: BackendRecord) -> NebiusConfig:
+        return NebiusConfig.__response__(
+            **json.loads(record.config),
+            creds=NebiusCreds.parse_raw(record.auth),
+        )

dstack/_internal/core/backends/nebius/models.py

@@ -0,0 +1,108 @@
+from typing import Annotated, Literal, Optional, Union
+
+from pydantic import Field, root_validator
+
+from dstack._internal.core.backends.base.models import fill_data
+from dstack._internal.core.models.common import CoreModel
+
+
+class NebiusServiceAccountCreds(CoreModel):
+    type: Annotated[Literal["service_account"], Field(description="The type of credentials")] = (
+        "service_account"
+    )
+    service_account_id: Annotated[str, Field(description="Service account ID")]
+    public_key_id: Annotated[str, Field(description="ID of the service account public key")]
+    private_key_file: Annotated[
+        Optional[str], Field(description=("Path to the service account private key"))
+    ] = None
+    private_key_content: Annotated[
+        str,
+        Field(
+            description=(
+                "Content of the service account private key. When configuring via"
+                " `server/config.yml`, it's automatically filled from `private_key_file`."
+                " When configuring via UI, it has to be specified explicitly."
+            )
+        ),
+    ]
+
+
+class NebiusServiceAccountFileCreds(CoreModel):
+    type: Annotated[Literal["service_account"], Field(description="The type of credentials")] = (
+        "service_account"
+    )
+    service_account_id: Annotated[str, Field(description="Service account ID")]
+    public_key_id: Annotated[str, Field(description="ID of the service account public key")]
+    private_key_file: Annotated[
+        Optional[str], Field(description=("Path to the service account private key"))
+    ] = None
+    private_key_content: Annotated[
+        Optional[str],
+        Field(
+            description=(
+                "Content of the service account private key. When configuring via"
+                " `server/config.yml`, it's automatically filled from `private_key_file`."
+                " When configuring via UI, it has to be specified explicitly."
+            )
+        ),
+    ] = None
+
+    @root_validator
+    def fill_data(cls, values):
+        return fill_data(
+            values, filename_field="private_key_file", data_field="private_key_content"
+        )
+
+
+AnyNebiusCreds = NebiusServiceAccountCreds
+NebiusCreds = AnyNebiusCreds
+AnyNebiusFileCreds = NebiusServiceAccountFileCreds
+
+
+class NebiusBackendConfig(CoreModel):
+    """
+    The backend config used in the API, server/config.yml, `NebiusConfigurator`.
+    It also serves as a base class for other backend config models.
+    Should not include creds.
+    """
+
+    type: Annotated[
+        Literal["nebius"],
+        Field(description="The type of backend"),
+    ] = "nebius"
+    regions: Annotated[
+        Optional[list[str]],
+        Field(description="The list of Nebius regions. Omit to use all regions"),
+    ] = None
+
+
+class NebiusBackendConfigWithCreds(NebiusBackendConfig):
+    """
+    Same as `NebiusBackendConfig` but also includes creds.
+    """
+
+    creds: Annotated[AnyNebiusCreds, Field(description="The credentials")]
+
+
+class NebiusBackendFileConfigWithCreds(NebiusBackendConfig):
+    creds: AnyNebiusFileCreds = Field(description="The credentials")
+
+
+AnyNebiusBackendConfig = Union[NebiusBackendConfig, NebiusBackendConfigWithCreds]
+
+
+class NebiusStoredConfig(NebiusBackendConfig):
+    """
+    The backend config used for config parameters in the DB.
+    Can extend `NebiusBackendConfig` with additional parameters.
+    """
+
+    pass
+
+
+class NebiusConfig(NebiusStoredConfig):
+    """
+    The backend config used by `NebiusBackend` and `NebiusCompute`.
+    """
+
+    creds: AnyNebiusCreds

dstack/_internal/core/backends/nebius/resources.py

@@ -0,0 +1,240 @@
+import logging
+import time
+from collections import defaultdict
+from collections.abc import Container as ContainerT
+from collections.abc import Generator
+from contextlib import contextmanager
+from tempfile import NamedTemporaryFile
+
+from nebius.aio.authorization.options import options_to_metadata
+from nebius.aio.operation import Operation as SDKOperation
+from nebius.aio.service_error import RequestError, StatusCode
+from nebius.aio.token.renewable import OPTION_RENEW_REQUEST_TIMEOUT, OPTION_RENEW_SYNCHRONOUS
+from nebius.api.nebius.common.v1 import Operation, ResourceMetadata
+from nebius.api.nebius.compute.v1 import (
+    AttachedDiskSpec,
+    CreateDiskRequest,
+    CreateInstanceRequest,
+    DeleteDiskRequest,
+    DeleteInstanceRequest,
+    DiskServiceClient,
+    DiskSpec,
+    ExistingDisk,
+    GetInstanceRequest,
+    Instance,
+    InstanceServiceClient,
+    InstanceSpec,
+    IPAddress,
+    NetworkInterfaceSpec,
+    PublicIPAddress,
+    ResourcesSpec,
+    SourceImageFamily,
+)
+from nebius.api.nebius.iam.v1 import (
+    Container,
+    ListProjectsRequest,
+    ListTenantsRequest,
+    ProjectServiceClient,
+    TenantServiceClient,
+)
+from nebius.api.nebius.vpc.v1 import ListSubnetsRequest, Subnet, SubnetServiceClient
+from nebius.sdk import SDK
+
+from dstack._internal.core.backends.nebius.models import NebiusServiceAccountCreds
+from dstack._internal.core.errors import BackendError, NoCapacityError
+from dstack._internal.utils.event_loop import DaemonEventLoop
+from dstack._internal.utils.logging import get_logger
+
+#
+# Guidelines on using the Nebius SDK:
+#
+# Do not use Request.wait() or other sync SDK methods, they suffer from deadlocks.
+# Instead, use async methods and await them with LOOP.await_()
+LOOP = DaemonEventLoop()
+# Pass a timeout to all methods to avoid infinite waiting
+REQUEST_TIMEOUT = 10
+# Pass REQUEST_MD to all methods to avoid infinite retries in case of invalid credentials
+REQUEST_MD = options_to_metadata(
+    {
+        OPTION_RENEW_SYNCHRONOUS: "true",
+        OPTION_RENEW_REQUEST_TIMEOUT: "5",
+    }
+)
+
+# disables log messages about errors such as invalid creds or expired timeouts
+logging.getLogger("nebius").setLevel(logging.CRITICAL)
+logger = get_logger(__name__)
+
+
+@contextmanager
+def wrap_capacity_errors() -> Generator[None, None, None]:
+    try:
+        yield
+    except RequestError as e:
+        if e.status.code == StatusCode.RESOURCE_EXHAUSTED or "Quota limit exceeded" in str(e):
+            raise NoCapacityError(e)
+        raise
+
+
+@contextmanager
+def ignore_errors(status_codes: ContainerT[StatusCode]) -> Generator[None, None, None]:
+    try:
+        yield
+    except RequestError as e:
+        if e.status.code not in status_codes:
+            raise
+
+
+def make_sdk(creds: NebiusServiceAccountCreds) -> SDK:
+    with NamedTemporaryFile("w") as f:
+        f.write(creds.private_key_content)
+        f.flush()
+        return SDK(
+            service_account_private_key_file_name=f.name,
+            service_account_public_key_id=creds.public_key_id,
+            service_account_id=creds.service_account_id,
+        )
+
+
+def wait_for_operation(
+    op: SDKOperation[Operation],
+    timeout: float,
+    interval: float = 1,
+) -> None:
+    # Re-implementation of SDKOperation.wait() to avoid https://github.com/nebius/pysdk/issues/74
+    deadline = time.monotonic() + timeout
+    while not op.done():
+        if time.monotonic() + interval > deadline:
+            raise TimeoutError(f"Operation {op.id} wait timeout")
+        time.sleep(interval)
+        LOOP.await_(op.update(timeout=REQUEST_TIMEOUT, metadata=REQUEST_MD))
+
+
+def get_region_to_project_id_map(sdk: SDK) -> dict[str, str]:
+    tenants = LOOP.await_(
+        TenantServiceClient(sdk).list(
+            ListTenantsRequest(), timeout=REQUEST_TIMEOUT, metadata=REQUEST_MD
+        )
+    )
+    if len(tenants.items) != 1:
+        raise ValueError(f"Expected to find 1 tenant, found {(len(tenants.items))}")
+    tenant_id = tenants.items[0].metadata.id
+    projects = LOOP.await_(
+        ProjectServiceClient(sdk).list(
+            ListProjectsRequest(parent_id=tenant_id, page_size=999),
+            timeout=REQUEST_TIMEOUT,
+            metadata=REQUEST_MD,
+        )
+    )
+    region_to_projects: defaultdict[str, list[Container]] = defaultdict(list)
+    for project in projects.items:
+        region_to_projects[project.status.region].append(project)
+    region_to_project_id = {}
+    for region, region_projects in region_to_projects.items():
+        if len(region_projects) != 1:
+            # Currently, there can only be one project per region.
+            # This condition is implemented just in case Nebius suddenly allows more projects.
+            region_projects = [
+                p for p in region_projects if p.metadata.name.startswith("default-project")
+            ]
+            if len(region_projects) != 1:
+                logger.warning(
+                    "Could not find the default project in region %s, tenant %s", region, tenant_id
+                )
+                continue
+        region_to_project_id[region] = region_projects[0].metadata.id
+    return region_to_project_id
+
+
+def get_default_subnet(sdk: SDK, project_id: str) -> Subnet:
+    subnets = LOOP.await_(
+        SubnetServiceClient(sdk).list(
+            ListSubnetsRequest(parent_id=project_id, page_size=999),
+            timeout=REQUEST_TIMEOUT,
+            metadata=REQUEST_MD,
+        )
+    )
+    for subnet in subnets.items:
+        if subnet.metadata.name.startswith("default-subnet"):
+            return subnet
+    raise BackendError(f"Could not find default subnet in project {project_id}")
+
+
+def create_disk(
+    sdk: SDK, name: str, project_id: str, size_mib: int, image_family: str
+) -> SDKOperation[Operation]:
+    client = DiskServiceClient(sdk)
+    request = CreateDiskRequest(
+        metadata=ResourceMetadata(
+            name=name,
+            parent_id=project_id,
+        ),
+        spec=DiskSpec(
+            size_mebibytes=size_mib,
+            type=DiskSpec.DiskType.NETWORK_SSD,
+            source_image_family=SourceImageFamily(image_family=image_family),
+        ),
+    )
+    with wrap_capacity_errors():
+        return LOOP.await_(client.create(request, timeout=REQUEST_TIMEOUT, metadata=REQUEST_MD))
+
+
+def delete_disk(sdk: SDK, disk_id: str) -> None:
+    LOOP.await_(
+        DiskServiceClient(sdk).delete(
+            DeleteDiskRequest(id=disk_id), timeout=REQUEST_TIMEOUT, metadata=REQUEST_MD
+        )
+    )
+
+
+def create_instance(
+    sdk: SDK,
+    name: str,
+    project_id: str,
+    user_data: str,
+    platform: str,
+    preset: str,
+    disk_id: str,
+    subnet_id: str,
+) -> SDKOperation[Operation]:
+    client = InstanceServiceClient(sdk)
+    request = CreateInstanceRequest(
+        metadata=ResourceMetadata(
+            name=name,
+            parent_id=project_id,
+        ),
+        spec=InstanceSpec(
+            cloud_init_user_data=user_data,
+            resources=ResourcesSpec(platform=platform, preset=preset),
+            boot_disk=AttachedDiskSpec(
+                attach_mode=AttachedDiskSpec.AttachMode.READ_WRITE,
+                existing_disk=ExistingDisk(id=disk_id),
+            ),
+            network_interfaces=[
+                NetworkInterfaceSpec(
+                    name="dstack-default-interface",
+                    subnet_id=subnet_id,
+                    ip_address=IPAddress(),
+                    public_ip_address=PublicIPAddress(static=True),
+                )
+            ],
+        ),
+    )
+    with wrap_capacity_errors():
+        return LOOP.await_(client.create(request, timeout=REQUEST_TIMEOUT, metadata=REQUEST_MD))
+
+
+def get_instance(sdk: SDK, instance_id: str) -> Instance:
+    return LOOP.await_(
+        InstanceServiceClient(sdk).get(
+            GetInstanceRequest(id=instance_id), timeout=REQUEST_TIMEOUT, metadata=REQUEST_MD
+        )
+    )
+
+
+def delete_instance(sdk: SDK, instance_id: str) -> SDKOperation[Operation]:
+    return LOOP.await_(
+        InstanceServiceClient(sdk).delete(
+            DeleteInstanceRequest(id=instance_id), timeout=REQUEST_TIMEOUT, metadata=REQUEST_MD
+        )
+    )