dstack 0.19.20__py3-none-any.whl → 0.19.22__py3-none-any.whl

dstack/_internal/cli/commands/apply.py

@@ -1,10 +1,10 @@
 import argparse
-from pathlib import Path
 
 from argcomplete import FilesCompleter
 
 from dstack._internal.cli.commands import APIBaseCommand
 from dstack._internal.cli.services.configurators import (
+    APPLY_STDIN_NAME,
     get_apply_configurator_class,
     load_apply_configuration,
 )
@@ -40,9 +40,12 @@ class ApplyCommand(APIBaseCommand):
         self._parser.add_argument(
             "-f",
             "--file",
-            type=Path,
             metavar="FILE",
-            help="The path to the configuration file. Defaults to [code]$PWD/.dstack.yml[/]",
+            help=(
+                "The path to the configuration file."
+                " Specify [code]-[/] to read configuration from stdin."
+                " Defaults to [code]$PWD/.dstack.yml[/]"
+            ),
             dest="configuration_file",
         ).completer = FilesCompleter(allowednames=["*.yml", "*.yaml"])
         self._parser.add_argument(
@@ -104,6 +107,8 @@ class ApplyCommand(APIBaseCommand):
                 return
 
             super()._command(args)
+            if not args.yes and args.configuration_file == APPLY_STDIN_NAME:
+                raise CLIError("Cannot read configuration from stdin if -y/--yes is not specified")
             if args.repo and args.no_repo:
                 raise CLIError("Either --repo or --no-repo can be specified")
             repo = None

dstack/_internal/cli/services/configurators/__init__.py

@@ -1,3 +1,4 @@
+import sys
 from pathlib import Path
 from typing import Dict, Optional, Tuple, Type
 
@@ -20,6 +21,9 @@ from dstack._internal.core.models.configurations import (
     parse_apply_configuration,
 )
 
+APPLY_STDIN_NAME = "-"
+
+
 apply_configurators_mapping: Dict[ApplyConfigurationType, Type[BaseApplyConfigurator]] = {
     cls.TYPE: cls
     for cls in [
@@ -62,6 +66,8 @@ def load_apply_configuration(
             raise ConfigurationError(
                 "No configuration file specified via `-f` and no default .dstack.yml configuration found"
             )
+    elif configuration_file == APPLY_STDIN_NAME:
+        configuration_path = sys.stdin.fileno()
     else:
         configuration_path = Path(configuration_file)
         if not configuration_path.exists():
@@ -71,4 +77,6 @@ def load_apply_configuration(
             conf = parse_apply_configuration(yaml.safe_load(f))
     except OSError:
         raise ConfigurationError(f"Failed to load configuration from {configuration_path}")
+    if isinstance(configuration_path, int):
+        return APPLY_STDIN_NAME, conf
     return str(configuration_path.absolute().relative_to(Path.cwd())), conf

dstack/_internal/cli/services/configurators/fleet.py

@@ -151,7 +151,7 @@ class FleetConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
                     time.sleep(LIVE_TABLE_PROVISION_INTERVAL_SECS)
                     fleet = self.api.client.fleets.get(self.api.project, fleet.name)
         except KeyboardInterrupt:
-            if confirm_ask("Delete the fleet before exiting?"):
+            if not command_args.yes and confirm_ask("Delete the fleet before exiting?"):
                 with console.status("Deleting fleet..."):
                     self.api.client.fleets.delete(
                         project_name=self.api.project, names=[fleet.name]

dstack/_internal/cli/services/configurators/gateway.py

@@ -121,7 +121,7 @@ class GatewayConfigurator(BaseApplyConfigurator):
                     time.sleep(LIVE_TABLE_PROVISION_INTERVAL_SECS)
                     gateway = self.api.client.gateways.get(self.api.project, gateway.name)
         except KeyboardInterrupt:
-            if confirm_ask("Delete the gateway before exiting?"):
+            if not command_args.yes and confirm_ask("Delete the gateway before exiting?"):
                 with console.status("Deleting gateway..."):
                     self.api.client.gateways.delete(
                         project_name=self.api.project,

dstack/_internal/cli/services/configurators/run.py

@@ -218,7 +218,9 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
                     exit(1)
         except KeyboardInterrupt:
             try:
-                if not confirm_ask(f"\nStop the run [code]{run.name}[/] before detaching?"):
+                if command_args.yes or not confirm_ask(
+                    f"\nStop the run [code]{run.name}[/] before detaching?"
+                ):
                     console.print("Detached")
                     abort_at_exit = False
                     return
@@ -339,6 +341,14 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
                     username=interpolator.interpolate_or_error(conf.registry_auth.username),
                     password=interpolator.interpolate_or_error(conf.registry_auth.password),
                 )
+            if isinstance(conf, ServiceConfiguration):
+                for probe in conf.probes:
+                    for header in probe.headers:
+                        header.value = interpolator.interpolate_or_error(header.value)
+                    if probe.url:
+                        probe.url = interpolator.interpolate_or_error(probe.url)
+                    if probe.body:
+                        probe.body = interpolator.interpolate_or_error(probe.body)
         except InterpolatorError as e:
             raise ConfigurationError(e.args[0])
 

dstack/_internal/cli/services/configurators/volume.py

@@ -110,7 +110,7 @@ class VolumeConfigurator(BaseApplyConfigurator):
                     time.sleep(LIVE_TABLE_PROVISION_INTERVAL_SECS)
                     volume = self.api.client.volumes.get(self.api.project, volume.name)
         except KeyboardInterrupt:
-            if confirm_ask("Delete the volume before exiting?"):
+            if not command_args.yes and confirm_ask("Delete the volume before exiting?"):
                 with console.status("Deleting volume..."):
                     self.api.client.volumes.delete(
                         project_name=self.api.project, names=[volume.name]

dstack/_internal/cli/utils/common.py

@@ -1,5 +1,6 @@
 import logging
-import os
+from datetime import datetime, timezone
+from pathlib import Path
 from typing import Any, Dict, Union
 
 from rich.console import Console
@@ -7,8 +8,10 @@ from rich.prompt import Confirm
 from rich.table import Table
 from rich.theme import Theme
 
+from dstack._internal import settings
 from dstack._internal.cli.utils.rich import DstackRichHandler
 from dstack._internal.core.errors import CLIError, DstackError
+from dstack._internal.utils.common import get_dstack_dir
 
 _colors = {
     "secondary": "grey58",
@@ -35,12 +38,52 @@ def cli_error(e: DstackError) -> CLIError:
     return CLIError(*e.args)
 
 
+def _get_cli_log_file() -> Path:
+    """Get the CLI log file path, rotating the previous log if needed."""
+    log_dir = get_dstack_dir() / "logs" / "cli"
+    log_file = log_dir / "latest.log"
+
+    if log_file.exists():
+        file_mtime = datetime.fromtimestamp(log_file.stat().st_mtime, tz=timezone.utc)
+        current_date = datetime.now(timezone.utc).date()
+
+        if file_mtime.date() < current_date:
+            date_str = file_mtime.strftime("%Y-%m-%d")
+            rotated_file = log_dir / f"{date_str}.log"
+
+            counter = 1
+            while rotated_file.exists():
+                rotated_file = log_dir / f"{date_str}-{counter}.log"
+                counter += 1
+
+            log_file.rename(rotated_file)
+
+    log_dir.mkdir(parents=True, exist_ok=True)
+    return log_file
+
+
 def configure_logging():
     dstack_logger = logging.getLogger("dstack")
-    dstack_logger.setLevel(os.getenv("DSTACK_CLI_LOG_LEVEL", "INFO").upper())
-    handler = DstackRichHandler(console=console)
-    handler.setFormatter(logging.Formatter(fmt="%(message)s", datefmt="[%X]"))
-    dstack_logger.addHandler(handler)
+    dstack_logger.handlers.clear()
+
+    log_file = _get_cli_log_file()
+
+    stdout_handler = DstackRichHandler(console=console)
+    stdout_handler.setFormatter(logging.Formatter(fmt="%(message)s", datefmt="[%X]"))
+    stdout_handler.setLevel(settings.CLI_LOG_LEVEL)
+    dstack_logger.addHandler(stdout_handler)
+
+    file_handler = logging.FileHandler(log_file)
+    file_handler.setFormatter(
+        logging.Formatter(
+            fmt="%(asctime)s - %(name)s - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
+        )
+    )
+    file_handler.setLevel(settings.CLI_FILE_LOG_LEVEL)
+    dstack_logger.addHandler(file_handler)
+
+    # the logger allows all messages, filtering is done by the handlers
+    dstack_logger.setLevel(logging.DEBUG)
 
 
 def confirm_ask(prompt, **kwargs) -> bool:

dstack/_internal/cli/utils/fleet.py

@@ -51,11 +51,11 @@ def get_fleets_table(
                 and total_blocks > 1
             ):
                 status = f"{busy_blocks}/{total_blocks} {InstanceStatus.BUSY.value}"
-            if (
-                instance.status in [InstanceStatus.IDLE, InstanceStatus.BUSY]
-                and instance.unreachable
-            ):
-                status += "\n(unreachable)"
+            if instance.status in [InstanceStatus.IDLE, InstanceStatus.BUSY]:
+                if instance.unreachable:
+                    status += "\n(unreachable)"
+                elif not instance.health_status.is_healthy():
+                    status += f"\n({instance.health_status.value})"
 
             backend = instance.backend or ""
             if backend == "remote":

dstack/_internal/cli/utils/run.py

@@ -12,11 +12,15 @@ from dstack._internal.core.models.profiles import (
     TerminationPolicy,
 )
 from dstack._internal.core.models.runs import (
+    JobStatus,
+    Probe,
+    ProbeSpec,
     RunPlan,
 )
 from dstack._internal.core.services.profiles import get_termination
 from dstack._internal.utils.common import (
     DateFormatter,
+    batched,
     format_duration_multiunit,
     format_pretty_duration,
     pretty_date,
@@ -156,6 +160,12 @@ def get_runs_table(
         table.add_column("INSTANCE TYPE", no_wrap=True, ratio=1)
     table.add_column("PRICE", style="grey58", ratio=1)
     table.add_column("STATUS", no_wrap=True, ratio=1)
+    if verbose or any(
+        run._run.is_deployment_in_progress()
+        and any(job.job_submissions[-1].probes for job in run._run.jobs)
+        for run in runs
+    ):
+        table.add_column("PROBES", ratio=1)
     table.add_column("SUBMITTED", style="grey58", no_wrap=True, ratio=1)
     if verbose:
         table.add_column("ERROR", no_wrap=True, ratio=2)
@@ -198,6 +208,9 @@ def get_runs_table(
                     else ""
                 ),
                 "STATUS": latest_job_submission.status_message,
+                "PROBES": _format_job_probes(
+                    job.job_spec.probes, latest_job_submission.probes, latest_job_submission.status
+                ),
                 "SUBMITTED": format_date(latest_job_submission.submitted_at),
                 "ERROR": latest_job_submission.error,
             }
@@ -226,3 +239,22 @@ def get_runs_table(
             add_row_from_dict(table, job_row, style="secondary" if len(run.jobs) != 1 else None)
 
     return table
+
+
+def _format_job_probes(
+    probe_specs: list[ProbeSpec], probes: list[Probe], job_status: JobStatus
+) -> str:
+    if not probes or job_status != JobStatus.RUNNING:
+        return ""
+    statuses = []
+    for probe_spec, probe in zip(probe_specs, probes):
+        # NOTE: the symbols are documented in concepts/services.md, keep in sync.
+        if probe.success_streak >= probe_spec.ready_after:
+            status = "[code]✓[/]"
+        elif probe.success_streak > 0:
+            status = "[warning]~[/]"
+        else:
+            status = "[error]×[/]"
+        statuses.append(status)
+    # split into whitespace-delimited batches to allow column wrapping
+    return " ".join("".join(batch) for batch in batched(statuses, 5))

dstack/_internal/core/backends/__init__.py

@@ -1,65 +0,0 @@
-from dstack._internal.core.backends.base.compute import (
-    ComputeWithCreateInstanceSupport,
-    ComputeWithGatewaySupport,
-    ComputeWithMultinodeSupport,
-    ComputeWithPlacementGroupSupport,
-    ComputeWithPrivateGatewaySupport,
-    ComputeWithReservationSupport,
-    ComputeWithVolumeSupport,
-)
-from dstack._internal.core.backends.base.configurator import Configurator
-from dstack._internal.core.backends.configurators import list_available_configurator_classes
-from dstack._internal.core.backends.local.compute import LocalCompute
-from dstack._internal.core.models.backends.base import BackendType
-from dstack._internal.settings import LOCAL_BACKEND_ENABLED
-
-
-def _get_backends_with_compute_feature(
-    configurator_classes: list[type[Configurator]],
-    compute_feature_class: type,
-) -> list[BackendType]:
-    backend_types_and_computes = [
-        (configurator_class.TYPE, configurator_class.BACKEND_CLASS.COMPUTE_CLASS)
-        for configurator_class in configurator_classes
-    ]
-    if LOCAL_BACKEND_ENABLED:
-        backend_types_and_computes.append((BackendType.LOCAL, LocalCompute))
-    backend_types = []
-    for backend_type, compute_class in backend_types_and_computes:
-        if issubclass(compute_class, compute_feature_class):
-            backend_types.append(backend_type)
-    return backend_types
-
-
-_configurator_classes = list_available_configurator_classes()
-
-
-# The following backend lists do not include unavailable backends (i.e. backends missing deps).
-BACKENDS_WITH_CREATE_INSTANCE_SUPPORT = _get_backends_with_compute_feature(
-    configurator_classes=_configurator_classes,
-    compute_feature_class=ComputeWithCreateInstanceSupport,
-)
-BACKENDS_WITH_MULTINODE_SUPPORT = [BackendType.REMOTE] + _get_backends_with_compute_feature(
-    configurator_classes=_configurator_classes,
-    compute_feature_class=ComputeWithMultinodeSupport,
-)
-BACKENDS_WITH_PLACEMENT_GROUPS_SUPPORT = _get_backends_with_compute_feature(
-    configurator_classes=_configurator_classes,
-    compute_feature_class=ComputeWithPlacementGroupSupport,
-)
-BACKENDS_WITH_RESERVATION_SUPPORT = _get_backends_with_compute_feature(
-    configurator_classes=_configurator_classes,
-    compute_feature_class=ComputeWithReservationSupport,
-)
-BACKENDS_WITH_GATEWAY_SUPPORT = _get_backends_with_compute_feature(
-    configurator_classes=_configurator_classes,
-    compute_feature_class=ComputeWithGatewaySupport,
-)
-BACKENDS_WITH_PRIVATE_GATEWAY_SUPPORT = _get_backends_with_compute_feature(
-    configurator_classes=_configurator_classes,
-    compute_feature_class=ComputeWithPrivateGatewaySupport,
-)
-BACKENDS_WITH_VOLUMES_SUPPORT = _get_backends_with_compute_feature(
-    configurator_classes=_configurator_classes,
-    compute_feature_class=ComputeWithVolumeSupport,
-)

dstack/_internal/core/backends/configurators.py

@@ -54,6 +54,15 @@ try:
 except ImportError:
     pass
 
+try:
+    from dstack._internal.core.backends.hotaisle.configurator import (
+        HotAisleConfigurator,
+    )
+
+    _CONFIGURATOR_CLASSES.append(HotAisleConfigurator)
+except ImportError:
+    pass
+
 try:
     from dstack._internal.core.backends.kubernetes.configurator import (
         KubernetesConfigurator,

dstack/_internal/core/backends/features.py

@@ -0,0 +1,64 @@
+from dstack._internal.core.backends.base.compute import (
+    ComputeWithCreateInstanceSupport,
+    ComputeWithGatewaySupport,
+    ComputeWithMultinodeSupport,
+    ComputeWithPlacementGroupSupport,
+    ComputeWithPrivateGatewaySupport,
+    ComputeWithReservationSupport,
+    ComputeWithVolumeSupport,
+)
+from dstack._internal.core.backends.base.configurator import Configurator
+from dstack._internal.core.backends.configurators import list_available_configurator_classes
+from dstack._internal.core.backends.local.compute import LocalCompute
+from dstack._internal.core.models.backends.base import BackendType
+from dstack._internal.settings import LOCAL_BACKEND_ENABLED
+
+_configurator_classes = list_available_configurator_classes()
+
+
+def _get_backends_with_compute_feature(
+    configurator_classes: list[type[Configurator]],
+    compute_feature_class: type,
+) -> list[BackendType]:
+    backend_types_and_computes = [
+        (configurator_class.TYPE, configurator_class.BACKEND_CLASS.COMPUTE_CLASS)
+        for configurator_class in configurator_classes
+    ]
+    if LOCAL_BACKEND_ENABLED:
+        backend_types_and_computes.append((BackendType.LOCAL, LocalCompute))
+    backend_types = []
+    for backend_type, compute_class in backend_types_and_computes:
+        if issubclass(compute_class, compute_feature_class):
+            backend_types.append(backend_type)
+    return backend_types
+
+
+# The following backend lists do not include unavailable backends (i.e. backends missing deps).
+BACKENDS_WITH_CREATE_INSTANCE_SUPPORT = _get_backends_with_compute_feature(
+    configurator_classes=_configurator_classes,
+    compute_feature_class=ComputeWithCreateInstanceSupport,
+)
+BACKENDS_WITH_MULTINODE_SUPPORT = [BackendType.REMOTE] + _get_backends_with_compute_feature(
+    configurator_classes=_configurator_classes,
+    compute_feature_class=ComputeWithMultinodeSupport,
+)
+BACKENDS_WITH_PLACEMENT_GROUPS_SUPPORT = _get_backends_with_compute_feature(
+    configurator_classes=_configurator_classes,
+    compute_feature_class=ComputeWithPlacementGroupSupport,
+)
+BACKENDS_WITH_RESERVATION_SUPPORT = _get_backends_with_compute_feature(
+    configurator_classes=_configurator_classes,
+    compute_feature_class=ComputeWithReservationSupport,
+)
+BACKENDS_WITH_GATEWAY_SUPPORT = _get_backends_with_compute_feature(
+    configurator_classes=_configurator_classes,
+    compute_feature_class=ComputeWithGatewaySupport,
+)
+BACKENDS_WITH_PRIVATE_GATEWAY_SUPPORT = _get_backends_with_compute_feature(
+    configurator_classes=_configurator_classes,
+    compute_feature_class=ComputeWithPrivateGatewaySupport,
+)
+BACKENDS_WITH_VOLUMES_SUPPORT = _get_backends_with_compute_feature(
+    configurator_classes=_configurator_classes,
+    compute_feature_class=ComputeWithVolumeSupport,
+)

dstack/_internal/core/backends/hotaisle/__init__.py

@@ -0,0 +1 @@
+# Hotaisle backend for dstack

dstack/_internal/core/backends/hotaisle/api_client.py

@@ -0,0 +1,109 @@
+from typing import Any, Dict, Optional
+
+import requests
+
+from dstack._internal.core.backends.base.configurator import raise_invalid_credentials_error
+from dstack._internal.utils.logging import get_logger
+
+API_URL = "https://admin.hotaisle.app/api"
+
+logger = get_logger(__name__)
+
+
+class HotAisleAPIClient:
+    def __init__(self, api_key: str, team_handle: str):
+        self.api_key = api_key
+        self.team_handle = team_handle
+
+    def validate_api_key(self) -> bool:
+        try:
+            self._validate_user_and_team()
+            return True
+        except requests.HTTPError as e:
+            if e.response.status_code == 401:
+                raise_invalid_credentials_error(
+                    fields=[["creds", "api_key"]], details="Invalid API key"
+                )
+            elif e.response.status_code == 403:
+                raise_invalid_credentials_error(
+                    fields=[["creds", "api_key"]],
+                    details="Authenticated user does note have required permissions",
+                )
+            raise e
+        except ValueError as e:
+            error_message = str(e)
+            if "No Hot Aisle teams found" in error_message:
+                raise_invalid_credentials_error(
+                    fields=[["creds", "api_key"]],
+                    details="Valid API key but no teams found for this user",
+                )
+            elif "not found" in error_message:
+                raise_invalid_credentials_error(
+                    fields=[["team_handle"]], details=f"Team handle '{self.team_handle}' not found"
+                )
+            raise e
+
+    def _validate_user_and_team(self) -> None:
+        url = f"{API_URL}/user/"
+        response = self._make_request("GET", url)
+        response.raise_for_status()
+        user_data = response.json()
+
+        teams = user_data.get("teams", [])
+        if not teams:
+            raise ValueError("No Hot Aisle teams found for this user")
+
+        available_teams = [team["handle"] for team in teams]
+        if self.team_handle not in available_teams:
+            raise ValueError(f"Hot Aisle team '{self.team_handle}' not found.")
+
+    def upload_ssh_key(self, public_key: str) -> bool:
+        url = f"{API_URL}/user/ssh_keys/"
+        payload = {"authorized_key": public_key}
+
+        response = self._make_request("POST", url, json=payload)
+
+        if response.status_code == 409:
+            return True  # Key already exists - success
+        response.raise_for_status()
+        return True
+
+    def create_virtual_machine(self, vm_payload: Dict[str, Any]) -> Dict[str, Any]:
+        url = f"{API_URL}/teams/{self.team_handle}/virtual_machines/"
+        response = self._make_request("POST", url, json=vm_payload)
+        response.raise_for_status()
+        vm_data = response.json()
+        return vm_data
+
+    def get_vm_state(self, vm_name: str) -> str:
+        url = f"{API_URL}/teams/{self.team_handle}/virtual_machines/{vm_name}/state/"
+        response = self._make_request("GET", url)
+        response.raise_for_status()
+        state_data = response.json()
+        return state_data["state"]
+
+    def terminate_virtual_machine(self, vm_name: str) -> None:
+        url = f"{API_URL}/teams/{self.team_handle}/virtual_machines/{vm_name}/"
+        response = self._make_request("DELETE", url)
+        if response.status_code == 404:
+            logger.debug("Hot Aisle virtual machine %s not found", vm_name)
+            return
+        response.raise_for_status()
+
+    def _make_request(
+        self, method: str, url: str, json: Optional[Dict[str, Any]] = None, timeout: int = 30
+    ) -> requests.Response:
+        headers = {
+            "accept": "application/json",
+            "Authorization": f"Token {self.api_key}",
+        }
+        if json is not None:
+            headers["Content-Type"] = "application/json"
+
+        return requests.request(
+            method=method,
+            url=url,
+            headers=headers,
+            json=json,
+            timeout=timeout,
+        )

dstack/_internal/core/backends/hotaisle/backend.py

@@ -0,0 +1,16 @@
+from dstack._internal.core.backends.base.backend import Backend
+from dstack._internal.core.backends.hotaisle.compute import HotAisleCompute
+from dstack._internal.core.backends.hotaisle.models import HotAisleConfig
+from dstack._internal.core.models.backends.base import BackendType
+
+
+class HotAisleBackend(Backend):
+    TYPE = BackendType.HOTAISLE
+    COMPUTE_CLASS = HotAisleCompute
+
+    def __init__(self, config: HotAisleConfig):
+        self.config = config
+        self._compute = HotAisleCompute(self.config)
+
+    def compute(self) -> HotAisleCompute:
+        return self._compute