dstack 0.19.1__py3-none-any.whl → 0.19.3__py3-none-any.whl

dstack/_internal/cli/commands/metrics.py

@@ -0,0 +1,138 @@
+import argparse
+import time
+from typing import Any, Dict, List, Optional, Union
+
+from rich.live import Live
+from rich.table import Table
+
+from dstack._internal.cli.commands import APIBaseCommand
+from dstack._internal.cli.services.completion import RunNameCompleter
+from dstack._internal.cli.utils.common import (
+    LIVE_TABLE_PROVISION_INTERVAL_SECS,
+    LIVE_TABLE_REFRESH_RATE_PER_SEC,
+    add_row_from_dict,
+    console,
+)
+from dstack._internal.core.errors import CLIError
+from dstack._internal.core.models.instances import Resources
+from dstack._internal.core.models.metrics import JobMetrics
+from dstack.api._public import Client
+from dstack.api._public.runs import Run
+
+
+class MetricsCommand(APIBaseCommand):
+    NAME = "metrics"
+    DESCRIPTION = "Show run metrics"
+
+    def _register(self):
+        super()._register()
+        self._parser.add_argument("run_name").completer = RunNameCompleter()
+        self._parser.add_argument(
+            "-w",
+            "--watch",
+            help="Watch run metrics in realtime",
+            action="store_true",
+        )
+
+    def _command(self, args: argparse.Namespace):
+        super()._command(args)
+        run = self.api.runs.get(run_name=args.run_name)
+        if run is None:
+            raise CLIError(f"Run {args.run_name} not found")
+        if run.status.is_finished():
+            raise CLIError(f"Run {args.run_name} is finished")
+        metrics = _get_run_jobs_metrics(api=self.api, run=run)
+
+        if not args.watch:
+            console.print(_get_metrics_table(run, metrics))
+            return
+
+        try:
+            with Live(console=console, refresh_per_second=LIVE_TABLE_REFRESH_RATE_PER_SEC) as live:
+                while True:
+                    live.update(_get_metrics_table(run, metrics))
+                    time.sleep(LIVE_TABLE_PROVISION_INTERVAL_SECS)
+                    run = self.api.runs.get(run_name=args.run_name)
+                    if run is None:
+                        raise CLIError(f"Run {args.run_name} not found")
+                    if run.status.is_finished():
+                        raise CLIError(f"Run {args.run_name} is finished")
+                    metrics = _get_run_jobs_metrics(api=self.api, run=run)
+        except KeyboardInterrupt:
+            pass
+
+
+def _get_run_jobs_metrics(api: Client, run: Run) -> List[JobMetrics]:
+    metrics = []
+    for job in run._run.jobs:
+        job_metrics = api.client.metrics.get_job_metrics(
+            project_name=api.project,
+            run_name=run.name,
+            replica_num=job.job_spec.replica_num,
+            job_num=job.job_spec.job_num,
+        )
+        metrics.append(job_metrics)
+    return metrics
+
+
+def _get_metrics_table(run: Run, metrics: List[JobMetrics]) -> Table:
+    table = Table(box=None)
+    table.add_column("NAME", style="bold", no_wrap=True)
+    table.add_column("CPU")
+    table.add_column("MEMORY")
+    table.add_column("GPU")
+
+    run_row: Dict[Union[str, int], Any] = {"NAME": run.name}
+    if len(run._run.jobs) != 1:
+        add_row_from_dict(table, run_row)
+
+    for job, job_metrics in zip(run._run.jobs, metrics):
+        jrd = job.job_submissions[-1].job_runtime_data
+        jpd = job.job_submissions[-1].job_provisioning_data
+        resources: Optional[Resources] = None
+        if jrd is not None and jrd.offer is not None:
+            resources = jrd.offer.instance.resources
+        elif jpd is not None:
+            resources = jpd.instance_type.resources
+        cpu_usage = _get_metric_value(job_metrics, "cpu_usage_percent")
+        if cpu_usage is not None:
+            if resources is not None:
+                cpu_usage = cpu_usage / resources.cpus
+            cpu_usage = f"{cpu_usage:.0f}%"
+        memory_usage = _get_metric_value(job_metrics, "memory_working_set_bytes")
+        if memory_usage is not None:
+            memory_usage = f"{round(memory_usage / 1024 / 1024)}MB"
+            if resources is not None:
+                memory_usage += f"/{resources.memory_mib}MB"
+        gpu_metrics = ""
+        gpus_detected_num = _get_metric_value(job_metrics, "gpus_detected_num")
+        if gpus_detected_num is not None:
+            for i in range(gpus_detected_num):
+                gpu_memory_usage = _get_metric_value(job_metrics, f"gpu_memory_usage_bytes_gpu{i}")
+                gpu_util_percent = _get_metric_value(job_metrics, f"gpu_util_percent_gpu{i}")
+                if gpu_memory_usage is not None:
+                    if i != 0:
+                        gpu_metrics += "\n"
+                    gpu_metrics += f"#{i} {round(gpu_memory_usage / 1024 / 1024)}MB"
+                    if resources is not None:
+                        gpu_metrics += f"/{resources.gpus[i].memory_mib}MB"
+                    gpu_metrics += f" {gpu_util_percent}% Util"
+
+        job_row: Dict[Union[str, int], Any] = {
+            "NAME": f"  replica={job.job_spec.replica_num} job={job.job_spec.job_num}",
+            "CPU": cpu_usage or "-",
+            "MEMORY": memory_usage or "-",
+            "GPU": gpu_metrics or "-",
+        }
+        if len(run._run.jobs) == 1:
+            job_row.update(run_row)
+        add_row_from_dict(table, job_row)
+
+    return table
+
+
+def _get_metric_value(job_metrics: JobMetrics, name: str) -> Optional[Any]:
+    for metric in job_metrics.metrics:
+        if metric.name == name:
+            return metric.values[-1]
+    return None

dstack/_internal/cli/commands/stats.py

@@ -1,128 +1,14 @@
 import argparse
-import time
-from typing import Any, Dict, List, Optional, Union
 
-from rich.live import Live
-from rich.table import Table
+from dstack._internal.cli.commands.metrics import MetricsCommand
+from dstack._internal.utils.logging import get_logger
 
-from dstack._internal.cli.commands import APIBaseCommand
-from dstack._internal.cli.services.completion import RunNameCompleter
-from dstack._internal.cli.utils.common import (
-    LIVE_TABLE_PROVISION_INTERVAL_SECS,
-    LIVE_TABLE_REFRESH_RATE_PER_SEC,
-    add_row_from_dict,
-    console,
-)
-from dstack._internal.core.errors import CLIError
-from dstack._internal.core.models.metrics import JobMetrics
-from dstack.api._public import Client
-from dstack.api._public.runs import Run
+logger = get_logger(__name__)
 
 
-class StatsCommand(APIBaseCommand):
+class StatsCommand(MetricsCommand):
     NAME = "stats"
-    DESCRIPTION = "Show run stats"
-
-    def _register(self):
-        super()._register()
-        self._parser.add_argument("run_name").completer = RunNameCompleter()
-        self._parser.add_argument(
-            "-w",
-            "--watch",
-            help="Watch run stats in realtime",
-            action="store_true",
-        )
 
     def _command(self, args: argparse.Namespace):
+        logger.warning("`dstack stats` is deprecated in favor of `dstack metrics`")
         super()._command(args)
-        run = self.api.runs.get(run_name=args.run_name)
-        if run is None:
-            raise CLIError(f"Run {args.run_name} not found")
-        if run.status.is_finished():
-            raise CLIError(f"Run {args.run_name} is finished")
-        metrics = _get_run_jobs_metrics(api=self.api, run=run)
-
-        if not args.watch:
-            console.print(_get_stats_table(run, metrics))
-            return
-
-        try:
-            with Live(console=console, refresh_per_second=LIVE_TABLE_REFRESH_RATE_PER_SEC) as live:
-                while True:
-                    live.update(_get_stats_table(run, metrics))
-                    time.sleep(LIVE_TABLE_PROVISION_INTERVAL_SECS)
-                    run = self.api.runs.get(run_name=args.run_name)
-                    if run is None:
-                        raise CLIError(f"Run {args.run_name} not found")
-                    if run.status.is_finished():
-                        raise CLIError(f"Run {args.run_name} is finished")
-                    metrics = _get_run_jobs_metrics(api=self.api, run=run)
-        except KeyboardInterrupt:
-            pass
-
-
-def _get_run_jobs_metrics(api: Client, run: Run) -> List[JobMetrics]:
-    metrics = []
-    for job in run._run.jobs:
-        job_metrics = api.client.metrics.get_job_metrics(
-            project_name=api.project,
-            run_name=run.name,
-            replica_num=job.job_spec.replica_num,
-            job_num=job.job_spec.job_num,
-        )
-        metrics.append(job_metrics)
-    return metrics
-
-
-def _get_stats_table(run: Run, metrics: List[JobMetrics]) -> Table:
-    table = Table(box=None)
-    table.add_column("NAME", style="bold", no_wrap=True)
-    table.add_column("CPU")
-    table.add_column("MEMORY")
-    table.add_column("GPU")
-
-    run_row: Dict[Union[str, int], Any] = {"NAME": run.name}
-    if len(run._run.jobs) != 1:
-        add_row_from_dict(table, run_row)
-
-    for job, job_metrics in zip(run._run.jobs, metrics):
-        cpu_usage = _get_metric_value(job_metrics, "cpu_usage_percent")
-        if cpu_usage is not None:
-            cpu_usage = f"{cpu_usage}%"
-        memory_usage = _get_metric_value(job_metrics, "memory_working_set_bytes")
-        if memory_usage is not None:
-            memory_usage = f"{round(memory_usage / 1024 / 1024)}MB"
-            if job.job_submissions[-1].job_provisioning_data is not None:
-                memory_usage += f"/{job.job_submissions[-1].job_provisioning_data.instance_type.resources.memory_mib}MB"
-        gpu_stats = ""
-        gpus_detected_num = _get_metric_value(job_metrics, "gpus_detected_num")
-        if gpus_detected_num is not None:
-            for i in range(gpus_detected_num):
-                gpu_memory_usage = _get_metric_value(job_metrics, f"gpu_memory_usage_bytes_gpu{i}")
-                gpu_util_percent = _get_metric_value(job_metrics, f"gpu_util_percent_gpu{i}")
-                if gpu_memory_usage is not None:
-                    if i != 0:
-                        gpu_stats += "\n"
-                    gpu_stats += f"#{i} {round(gpu_memory_usage / 1024 / 1024)}MB"
-                    if job.job_submissions[-1].job_provisioning_data is not None:
-                        gpu_stats += f"/{job.job_submissions[-1].job_provisioning_data.instance_type.resources.gpus[i].memory_mib}MB"
-                    gpu_stats += f" {gpu_util_percent}% Util"
-
-        job_row: Dict[Union[str, int], Any] = {
-            "NAME": f"  replica={job.job_spec.replica_num} job={job.job_spec.job_num}",
-            "CPU": cpu_usage or "-",
-            "MEMORY": memory_usage or "-",
-            "GPU": gpu_stats or "-",
-        }
-        if len(run._run.jobs) == 1:
-            job_row.update(run_row)
-        add_row_from_dict(table, job_row)
-
-    return table
-
-
-def _get_metric_value(job_metrics: JobMetrics, name: str) -> Optional[Any]:
-    for metric in job_metrics.metrics:
-        if metric.name == name:
-            return metric.values[-1]
-    return None

dstack/_internal/cli/main.py

@@ -13,6 +13,7 @@ from dstack._internal.cli.commands.fleet import FleetCommand
 from dstack._internal.cli.commands.gateway import GatewayCommand
 from dstack._internal.cli.commands.init import InitCommand
 from dstack._internal.cli.commands.logs import LogsCommand
+from dstack._internal.cli.commands.metrics import MetricsCommand
 from dstack._internal.cli.commands.ps import PsCommand
 from dstack._internal.cli.commands.server import ServerCommand
 from dstack._internal.cli.commands.stats import StatsCommand
@@ -65,6 +66,7 @@ def main():
     GatewayCommand.register(subparsers)
     InitCommand.register(subparsers)
     LogsCommand.register(subparsers)
+    MetricsCommand.register(subparsers)
     PsCommand.register(subparsers)
     ServerCommand.register(subparsers)
     StatsCommand.register(subparsers)

dstack/_internal/cli/services/profile.py

@@ -65,6 +65,13 @@ def register_profile_args(parser: argparse.ArgumentParser):
     )
 
     fleets_group = parser.add_argument_group("Fleets")
+    fleets_group.add_argument(
+        "--fleet",
+        action="append",
+        metavar="NAME",
+        dest="fleets",
+        help="Consider only instances from the specified fleet(s) for reuse",
+    )
     fleets_group_exc = fleets_group.add_mutually_exclusive_group()
     fleets_group_exc.add_argument(
         "-R",
@@ -147,6 +154,8 @@ def apply_profile_args(
     if args.max_duration is not None:
         profile_settings.max_duration = args.max_duration
 
+    if args.fleets:
+        profile_settings.fleets = args.fleets
     if args.idle_duration is not None:
         profile_settings.idle_duration = args.idle_duration
     elif args.dont_destroy:

dstack/_internal/core/backends/aws/configurator.py

@@ -34,6 +34,7 @@ from dstack._internal.utils.logging import get_logger
 
 logger = get_logger(__name__)
 
+# where dstack OS images are published
 REGIONS = [
     ("US East, N. Virginia", "us-east-1"),
     ("US East, Ohio", "us-east-2"),

dstack/_internal/core/backends/base/compute.py

@@ -94,6 +94,9 @@ class Compute(ABC):
         """
         Terminates an instance by `instance_id`.
         If the instance does not exist, it should not raise errors but return silently.
+
+        Should return ASAP. If required to wait for some operation, raise `NotYetTerminated`.
+        In this case, the method will be called again after a few seconds.
         """
         pass
 
@@ -525,7 +528,7 @@ def get_run_shim_script(is_privileged: bool, pjrt_device: Optional[str]) -> List
     pjrt_device_env = f"--pjrt-device={pjrt_device}" if pjrt_device else ""
 
     return [
-        f"nohup dstack-shim {privileged_flag} {pjrt_device_env} &",
+        f"nohup {DSTACK_SHIM_BINARY_PATH} {privileged_flag} {pjrt_device_env} &",
     ]
 
 

dstack/_internal/core/backends/base/models.py

@@ -1,14 +1,14 @@
 from pathlib import Path
 
 
-def fill_data(values: dict):
-    if values.get("data") is not None:
+def fill_data(values: dict, filename_field: str = "filename", data_field: str = "data") -> dict:
+    if values.get(data_field) is not None:
         return values
-    if "filename" not in values:
-        raise ValueError()
+    if (filename := values.get(filename_field)) is None:
+        raise ValueError(f"Either `{filename_field}` or `{data_field}` must be specified")
     try:
-        with open(Path(values["filename"]).expanduser()) as f:
-            values["data"] = f.read()
+        with open(Path(filename).expanduser()) as f:
+            values[data_field] = f.read()
     except OSError:
-        raise ValueError(f"No such file {values['filename']}")
+        raise ValueError(f"No such file {filename}")
     return values

dstack/_internal/core/backends/configurators.py

@@ -63,6 +63,15 @@ try:
 except ImportError:
     pass
 
+try:
+    from dstack._internal.core.backends.nebius.configurator import (
+        NebiusConfigurator,
+    )
+
+    _CONFIGURATOR_CLASSES.append(NebiusConfigurator)
+except ImportError:
+    pass
+
 try:
     from dstack._internal.core.backends.oci.configurator import OCIConfigurator
 

dstack/_internal/core/backends/cudo/compute.py

@@ -41,6 +41,7 @@ class CudoCompute(
     ) -> List[InstanceOfferWithAvailability]:
         offers = get_catalog_offers(
             backend=BackendType.CUDO,
+            locations=self.config.regions,
             requirements=requirements,
         )
         offers = [
@@ -48,6 +49,7 @@ class CudoCompute(
                 **offer.dict(), availability=InstanceAvailability.AVAILABLE
             )
             for offer in offers
+            # in-hyderabad-1 is known to have provisioning issues
             if offer.region not in ["in-hyderabad-1"]
         ]
         return offers

dstack/_internal/core/backends/cudo/configurator.py

@@ -17,17 +17,6 @@ from dstack._internal.core.backends.cudo.models import (
 )
 from dstack._internal.core.models.backends.base import BackendType
 
-REGIONS = [
-    "no-luster-1",
-    "se-smedjebacken-1",
-    "gb-london-1",
-    "se-stockholm-1",
-    "us-newyork-1",
-    "us-santaclara-1",
-]
-
-DEFAULT_REGION = "no-luster-1"
-
 
 class CudoConfigurator(Configurator):
     TYPE = BackendType.CUDO
@@ -39,8 +28,6 @@ class CudoConfigurator(Configurator):
     def create_backend(
         self, project_name: str, config: CudoBackendConfigWithCreds
     ) -> BackendRecord:
-        if config.regions is None:
-            config.regions = REGIONS
         return BackendRecord(
             config=CudoStoredConfig(
                 **CudoBackendConfig.__response__.parse_obj(config).dict()

dstack/_internal/core/backends/datacrunch/compute.py

@@ -1,5 +1,9 @@
 from typing import Dict, List, Optional
 
+from datacrunch import DataCrunchClient
+from datacrunch.exceptions import APIException
+from datacrunch.instances.instances import Instance
+
 from dstack._internal.core.backends.base.backend import Compute
 from dstack._internal.core.backends.base.compute import (
     ComputeWithCreateInstanceSupport,
@@ -7,8 +11,8 @@ from dstack._internal.core.backends.base.compute import (
     get_shim_commands,
 )
 from dstack._internal.core.backends.base.offers import get_catalog_offers
-from dstack._internal.core.backends.datacrunch.api_client import DataCrunchAPIClient
 from dstack._internal.core.backends.datacrunch.models import DataCrunchConfig
+from dstack._internal.core.errors import NoCapacityError
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.instances import (
     InstanceAvailability,
@@ -19,14 +23,12 @@ from dstack._internal.core.models.instances import (
 from dstack._internal.core.models.resources import Memory, Range
 from dstack._internal.core.models.runs import JobProvisioningData, Requirements
 from dstack._internal.utils.logging import get_logger
+from dstack._internal.utils.ssh import get_public_key_fingerprint
 
 logger = get_logger("datacrunch.compute")
 
 MAX_INSTANCE_NAME_LEN = 60
 
-# Ubuntu 22.04 + CUDA 12.0 + Docker
-# from API https://datacrunch.stoplight.io/docs/datacrunch-public/c46ab45dbc508-get-all-image-types
-IMAGE_ID = "2088da25-bb0d-41cc-a191-dccae45d96fd"
 IMAGE_SIZE = Memory.parse("50GB")
 
 CONFIGURABLE_DISK_SIZE = Range[Memory](min=IMAGE_SIZE, max=None)
@@ -39,7 +41,10 @@ class DataCrunchCompute(
     def __init__(self, config: DataCrunchConfig):
         super().__init__()
         self.config = config
-        self.api_client = DataCrunchAPIClient(config.creds.client_id, config.creds.client_secret)
+        self.client = DataCrunchClient(
+            client_id=self.config.creds.client_id,
+            client_secret=self.config.creds.client_secret,
+        )
 
     def get_offers(
         self, requirements: Optional[Requirements] = None
@@ -56,14 +61,12 @@ class DataCrunchCompute(
     def _get_offers_with_availability(
         self, offers: List[InstanceOffer]
     ) -> List[InstanceOfferWithAvailability]:
-        raw_availabilities: List[Dict] = self.api_client.client.instances.get_availabilities()
+        raw_availabilities: List[Dict] = self.client.instances.get_availabilities()
 
         region_availabilities = {}
         for location in raw_availabilities:
             location_code = location["location_code"]
             availabilities = location["availabilities"]
-            if location_code not in self.config.regions:
-                continue
             for name in availabilities:
                 key = (name, location_code)
                 region_availabilities[key] = InstanceAvailability.AVAILABLE
@@ -91,50 +94,50 @@ class DataCrunchCompute(
         for ssh_public_key in public_keys:
             ssh_ids.append(
                 # datacrunch allows you to use the same name
-                self.api_client.get_or_create_ssh_key(
+                _get_or_create_ssh_key(
+                    client=self.client,
                     name=f"dstack-{instance_config.instance_name}.key",
                     public_key=ssh_public_key,
                 )
             )
 
         commands = get_shim_commands(authorized_keys=public_keys)
-
         startup_script = " ".join([" && ".join(commands)])
         script_name = f"dstack-{instance_config.instance_name}.sh"
-
-        logger.debug("startup script:", startup_script)
-
-        startup_script_ids = self.api_client.get_or_create_startup_scrpit(
-            name=script_name, script=startup_script
+        startup_script_ids = _get_or_create_startup_scrpit(
+            client=self.client,
+            name=script_name,
+            script=startup_script,
         )
 
         disk_size = round(instance_offer.instance.resources.disk.size_mib / 1024)
-
-        instance = self.api_client.deploy_instance(
-            instance_type=instance_offer.instance.name,
-            ssh_key_ids=ssh_ids,
-            startup_script_id=startup_script_ids,
-            hostname=instance_name,
-            description=instance_name,
-            image=IMAGE_ID,
-            disk_size=disk_size,
-            location=instance_offer.region,
-        )
+        image_id = _get_vm_image_id(instance_offer)
 
         logger.debug(
-            "deploy_instance",
+            "Deploying datacrunch instance",
             {
                 "instance_type": instance_offer.instance.name,
                 "ssh_key_ids": ssh_ids,
                 "startup_script_id": startup_script_ids,
                 "hostname": instance_name,
                 "description": instance_name,
-                "image": IMAGE_ID,
+                "image": image_id,
                 "disk_size": disk_size,
                 "location": instance_offer.region,
             },
         )
-
+        instance = _deploy_instance(
+            client=self.client,
+            instance_type=instance_offer.instance.name,
+            ssh_key_ids=ssh_ids,
+            startup_script_id=startup_script_ids,
+            hostname=instance_name,
+            description=instance_name,
+            image=image_id,
+            disk_size=disk_size,
+            is_spot=instance_offer.instance.resources.spot,
+            location=instance_offer.region,
+        )
         return JobProvisioningData(
             backend=instance_offer.backend,
             instance_type=instance_offer.instance,
@@ -152,8 +155,14 @@ class DataCrunchCompute(
 
     def terminate_instance(
         self, instance_id: str, region: str, backend_data: Optional[str] = None
-    ) -> None:
-        self.api_client.delete_instance(instance_id)
+    ):
+        try:
+            self.client.instances.action(id_list=[instance_id], action="delete")
+        except APIException as e:
+            if e.message == "Invalid instance id":
+                logger.debug("Skipping instance %s termination. Instance not found.", instance_id)
+                return
+            raise
 
     def update_provisioning_data(
         self,
@@ -161,6 +170,83 @@ class DataCrunchCompute(
         project_ssh_public_key: str,
         project_ssh_private_key: str,
     ):
-        instance = self.api_client.get_instance_by_id(provisioning_data.instance_id)
+        instance = _get_instance_by_id(self.client, provisioning_data.instance_id)
         if instance is not None and instance.status == "running":
             provisioning_data.hostname = instance.ip
+
+
+def _get_vm_image_id(instance_offer: InstanceOfferWithAvailability) -> str:
+    # https://api.datacrunch.io/v1/images
+    if (
+        len(instance_offer.instance.resources.gpus) > 0
+        and instance_offer.instance.resources.gpus[0].name == "V100"
+    ):
+        # Ubuntu 22.04 + CUDA 12.0 + Docker
+        return "2088da25-bb0d-41cc-a191-dccae45d96fd"
+    # Ubuntu 24.04 + CUDA 12.8 Open + Docker
+    return "77777777-4f48-4249-82b3-f199fb9b701b"
+
+
+def _get_or_create_ssh_key(client: DataCrunchClient, name: str, public_key: str) -> str:
+    fingerprint = get_public_key_fingerprint(public_key)
+    keys = client.ssh_keys.get()
+    found_keys = [key for key in keys if fingerprint == get_public_key_fingerprint(key.public_key)]
+    if found_keys:
+        key = found_keys[0]
+        return key.id
+    key = client.ssh_keys.create(name, public_key)
+    return key.id
+
+
+def _get_or_create_startup_scrpit(client: DataCrunchClient, name: str, script: str) -> str:
+    scripts = client.startup_scripts.get()
+    found_scripts = [startup_script for startup_script in scripts if script == startup_script]
+    if found_scripts:
+        startup_script = found_scripts[0]
+        return startup_script.id
+
+    startup_script = client.startup_scripts.create(name, script)
+    return startup_script.id
+
+
+def _get_instance_by_id(
+    client: DataCrunchClient,
+    instance_id: str,
+) -> Optional[Instance]:
+    try:
+        return client.instances.get_by_id(instance_id)
+    except APIException as e:
+        if e.message == "Invalid instance id":
+            return None
+        raise
+
+
+def _deploy_instance(
+    client: DataCrunchClient,
+    instance_type: str,
+    image: str,
+    ssh_key_ids: List[str],
+    hostname: str,
+    description: str,
+    startup_script_id: str,
+    disk_size: int,
+    is_spot: bool,
+    location: str,
+) -> Instance:
+    try:
+        instance = client.instances.create(
+            instance_type=instance_type,
+            image=image,
+            ssh_key_ids=ssh_key_ids,
+            hostname=hostname,
+            description=description,
+            startup_script_id=startup_script_id,
+            is_spot=is_spot,
+            location=location,
+            os_volume={"name": "OS volume", "size": disk_size},
+        )
+    except APIException as e:
+        # FIXME: Catch only no capacity errors
+        raise NoCapacityError(f"DataCrunch API error: {e.message}")
+
+    return instance