dstack 0.19.21__py3-none-any.whl → 0.19.23rc1__py3-none-any.whl

dstack/_internal/cli/commands/apply.py

@@ -1,10 +1,10 @@
 import argparse
-from pathlib import Path
 
 from argcomplete import FilesCompleter
 
 from dstack._internal.cli.commands import APIBaseCommand
 from dstack._internal.cli.services.configurators import (
+    APPLY_STDIN_NAME,
     get_apply_configurator_class,
     load_apply_configuration,
 )
@@ -40,9 +40,12 @@ class ApplyCommand(APIBaseCommand):
         self._parser.add_argument(
             "-f",
             "--file",
-            type=Path,
             metavar="FILE",
-            help="The path to the configuration file. Defaults to [code]$PWD/.dstack.yml[/]",
+            help=(
+                "The path to the configuration file."
+                " Specify [code]-[/] to read configuration from stdin."
+                " Defaults to [code]$PWD/.dstack.yml[/]"
+            ),
             dest="configuration_file",
         ).completer = FilesCompleter(allowednames=["*.yml", "*.yaml"])
         self._parser.add_argument(
@@ -104,6 +107,8 @@ class ApplyCommand(APIBaseCommand):
                 return
 
             super()._command(args)
+            if not args.yes and args.configuration_file == APPLY_STDIN_NAME:
+                raise CLIError("Cannot read configuration from stdin if -y/--yes is not specified")
             if args.repo and args.no_repo:
                 raise CLIError("Either --repo or --no-repo can be specified")
             repo = None

dstack/_internal/cli/services/configurators/__init__.py

@@ -1,3 +1,4 @@
+import sys
 from pathlib import Path
 from typing import Dict, Optional, Tuple, Type
 
@@ -20,6 +21,9 @@ from dstack._internal.core.models.configurations import (
     parse_apply_configuration,
 )
 
+APPLY_STDIN_NAME = "-"
+
+
 apply_configurators_mapping: Dict[ApplyConfigurationType, Type[BaseApplyConfigurator]] = {
     cls.TYPE: cls
     for cls in [
@@ -62,6 +66,8 @@ def load_apply_configuration(
             raise ConfigurationError(
                 "No configuration file specified via `-f` and no default .dstack.yml configuration found"
             )
+    elif configuration_file == APPLY_STDIN_NAME:
+        configuration_path = sys.stdin.fileno()
     else:
         configuration_path = Path(configuration_file)
         if not configuration_path.exists():
@@ -71,4 +77,6 @@ def load_apply_configuration(
             conf = parse_apply_configuration(yaml.safe_load(f))
     except OSError:
         raise ConfigurationError(f"Failed to load configuration from {configuration_path}")
+    if isinstance(configuration_path, int):
+        return APPLY_STDIN_NAME, conf
     return str(configuration_path.absolute().relative_to(Path.cwd())), conf

dstack/_internal/cli/services/configurators/fleet.py

@@ -151,7 +151,7 @@ class FleetConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
                     time.sleep(LIVE_TABLE_PROVISION_INTERVAL_SECS)
                     fleet = self.api.client.fleets.get(self.api.project, fleet.name)
         except KeyboardInterrupt:
-            if confirm_ask("Delete the fleet before exiting?"):
+            if not command_args.yes and confirm_ask("Delete the fleet before exiting?"):
                 with console.status("Deleting fleet..."):
                     self.api.client.fleets.delete(
                         project_name=self.api.project, names=[fleet.name]

dstack/_internal/cli/services/configurators/gateway.py

@@ -121,7 +121,7 @@ class GatewayConfigurator(BaseApplyConfigurator):
                     time.sleep(LIVE_TABLE_PROVISION_INTERVAL_SECS)
                     gateway = self.api.client.gateways.get(self.api.project, gateway.name)
         except KeyboardInterrupt:
-            if confirm_ask("Delete the gateway before exiting?"):
+            if not command_args.yes and confirm_ask("Delete the gateway before exiting?"):
                 with console.status("Deleting gateway..."):
                     self.api.client.gateways.delete(
                         project_name=self.api.project,

dstack/_internal/cli/services/configurators/run.py

@@ -218,7 +218,9 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
                     exit(1)
         except KeyboardInterrupt:
             try:
-                if not confirm_ask(f"\nStop the run [code]{run.name}[/] before detaching?"):
+                if command_args.yes or not confirm_ask(
+                    f"\nStop the run [code]{run.name}[/] before detaching?"
+                ):
                     console.print("Detached")
                     abort_at_exit = False
                     return
@@ -339,6 +341,14 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
                     username=interpolator.interpolate_or_error(conf.registry_auth.username),
                     password=interpolator.interpolate_or_error(conf.registry_auth.password),
                 )
+            if isinstance(conf, ServiceConfiguration):
+                for probe in conf.probes:
+                    for header in probe.headers:
+                        header.value = interpolator.interpolate_or_error(header.value)
+                    if probe.url:
+                        probe.url = interpolator.interpolate_or_error(probe.url)
+                    if probe.body:
+                        probe.body = interpolator.interpolate_or_error(probe.body)
         except InterpolatorError as e:
             raise ConfigurationError(e.args[0])
 

dstack/_internal/cli/services/configurators/volume.py

@@ -110,7 +110,7 @@ class VolumeConfigurator(BaseApplyConfigurator):
                     time.sleep(LIVE_TABLE_PROVISION_INTERVAL_SECS)
                     volume = self.api.client.volumes.get(self.api.project, volume.name)
         except KeyboardInterrupt:
-            if confirm_ask("Delete the volume before exiting?"):
+            if not command_args.yes and confirm_ask("Delete the volume before exiting?"):
                 with console.status("Deleting volume..."):
                     self.api.client.volumes.delete(
                         project_name=self.api.project, names=[volume.name]

dstack/_internal/cli/utils/common.py

@@ -1,5 +1,6 @@
 import logging
-import os
+from datetime import datetime, timezone
+from pathlib import Path
 from typing import Any, Dict, Union
 
 from rich.console import Console
@@ -7,8 +8,10 @@ from rich.prompt import Confirm
 from rich.table import Table
 from rich.theme import Theme
 
+from dstack._internal import settings
 from dstack._internal.cli.utils.rich import DstackRichHandler
 from dstack._internal.core.errors import CLIError, DstackError
+from dstack._internal.utils.common import get_dstack_dir
 
 _colors = {
     "secondary": "grey58",
@@ -35,12 +38,52 @@ def cli_error(e: DstackError) -> CLIError:
     return CLIError(*e.args)
 
 
+def _get_cli_log_file() -> Path:
+    """Get the CLI log file path, rotating the previous log if needed."""
+    log_dir = get_dstack_dir() / "logs" / "cli"
+    log_file = log_dir / "latest.log"
+
+    if log_file.exists():
+        file_mtime = datetime.fromtimestamp(log_file.stat().st_mtime, tz=timezone.utc)
+        current_date = datetime.now(timezone.utc).date()
+
+        if file_mtime.date() < current_date:
+            date_str = file_mtime.strftime("%Y-%m-%d")
+            rotated_file = log_dir / f"{date_str}.log"
+
+            counter = 1
+            while rotated_file.exists():
+                rotated_file = log_dir / f"{date_str}-{counter}.log"
+                counter += 1
+
+            log_file.rename(rotated_file)
+
+    log_dir.mkdir(parents=True, exist_ok=True)
+    return log_file
+
+
 def configure_logging():
     dstack_logger = logging.getLogger("dstack")
-    dstack_logger.setLevel(os.getenv("DSTACK_CLI_LOG_LEVEL", "INFO").upper())
-    handler = DstackRichHandler(console=console)
-    handler.setFormatter(logging.Formatter(fmt="%(message)s", datefmt="[%X]"))
-    dstack_logger.addHandler(handler)
+    dstack_logger.handlers.clear()
+
+    log_file = _get_cli_log_file()
+
+    stdout_handler = DstackRichHandler(console=console)
+    stdout_handler.setFormatter(logging.Formatter(fmt="%(message)s", datefmt="[%X]"))
+    stdout_handler.setLevel(settings.CLI_LOG_LEVEL)
+    dstack_logger.addHandler(stdout_handler)
+
+    file_handler = logging.FileHandler(log_file)
+    file_handler.setFormatter(
+        logging.Formatter(
+            fmt="%(asctime)s - %(name)s - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
+        )
+    )
+    file_handler.setLevel(settings.CLI_FILE_LOG_LEVEL)
+    dstack_logger.addHandler(file_handler)
+
+    # the logger allows all messages, filtering is done by the handlers
+    dstack_logger.setLevel(logging.DEBUG)
 
 
 def confirm_ask(prompt, **kwargs) -> bool:

dstack/_internal/cli/utils/fleet.py

@@ -51,11 +51,11 @@ def get_fleets_table(
                 and total_blocks > 1
             ):
                 status = f"{busy_blocks}/{total_blocks} {InstanceStatus.BUSY.value}"
-            if (
-                instance.status in [InstanceStatus.IDLE, InstanceStatus.BUSY]
-                and instance.unreachable
-            ):
-                status += "\n(unreachable)"
+            if instance.status in [InstanceStatus.IDLE, InstanceStatus.BUSY]:
+                if instance.unreachable:
+                    status += "\n(unreachable)"
+                elif not instance.health_status.is_healthy():
+                    status += f"\n({instance.health_status.value})"
 
             backend = instance.backend or ""
             if backend == "remote":

dstack/_internal/cli/utils/run.py

@@ -12,11 +12,15 @@ from dstack._internal.core.models.profiles import (
     TerminationPolicy,
 )
 from dstack._internal.core.models.runs import (
+    JobStatus,
+    Probe,
+    ProbeSpec,
     RunPlan,
 )
 from dstack._internal.core.services.profiles import get_termination
 from dstack._internal.utils.common import (
     DateFormatter,
+    batched,
     format_duration_multiunit,
     format_pretty_duration,
     pretty_date,
@@ -156,6 +160,12 @@ def get_runs_table(
         table.add_column("INSTANCE TYPE", no_wrap=True, ratio=1)
     table.add_column("PRICE", style="grey58", ratio=1)
     table.add_column("STATUS", no_wrap=True, ratio=1)
+    if verbose or any(
+        run._run.is_deployment_in_progress()
+        and any(job.job_submissions[-1].probes for job in run._run.jobs)
+        for run in runs
+    ):
+        table.add_column("PROBES", ratio=1)
     table.add_column("SUBMITTED", style="grey58", no_wrap=True, ratio=1)
     if verbose:
         table.add_column("ERROR", no_wrap=True, ratio=2)
@@ -198,6 +208,9 @@ def get_runs_table(
                     else ""
                 ),
                 "STATUS": latest_job_submission.status_message,
+                "PROBES": _format_job_probes(
+                    job.job_spec.probes, latest_job_submission.probes, latest_job_submission.status
+                ),
                 "SUBMITTED": format_date(latest_job_submission.submitted_at),
                 "ERROR": latest_job_submission.error,
             }
@@ -226,3 +239,22 @@ def get_runs_table(
             add_row_from_dict(table, job_row, style="secondary" if len(run.jobs) != 1 else None)
 
     return table
+
+
+def _format_job_probes(
+    probe_specs: list[ProbeSpec], probes: list[Probe], job_status: JobStatus
+) -> str:
+    if not probes or job_status != JobStatus.RUNNING:
+        return ""
+    statuses = []
+    for probe_spec, probe in zip(probe_specs, probes):
+        # NOTE: the symbols are documented in concepts/services.md, keep in sync.
+        if probe.success_streak >= probe_spec.ready_after:
+            status = "[code]✓[/]"
+        elif probe.success_streak > 0:
+            status = "[warning]~[/]"
+        else:
+            status = "[error]×[/]"
+        statuses.append(status)
+    # split into whitespace-delimited batches to allow column wrapping
+    return " ".join("".join(batch) for batch in batched(statuses, 5))

dstack/_internal/core/backends/configurators.py

@@ -54,6 +54,15 @@ try:
 except ImportError:
     pass
 
+try:
+    from dstack._internal.core.backends.hotaisle.configurator import (
+        HotAisleConfigurator,
+    )
+
+    _CONFIGURATOR_CLASSES.append(HotAisleConfigurator)
+except ImportError:
+    pass
+
 try:
     from dstack._internal.core.backends.kubernetes.configurator import (
         KubernetesConfigurator,

dstack/_internal/core/backends/hotaisle/__init__.py

@@ -0,0 +1 @@
+# Hotaisle backend for dstack

dstack/_internal/core/backends/hotaisle/api_client.py

@@ -0,0 +1,109 @@
+from typing import Any, Dict, Optional
+
+import requests
+
+from dstack._internal.core.backends.base.configurator import raise_invalid_credentials_error
+from dstack._internal.utils.logging import get_logger
+
+API_URL = "https://admin.hotaisle.app/api"
+
+logger = get_logger(__name__)
+
+
+class HotAisleAPIClient:
+    def __init__(self, api_key: str, team_handle: str):
+        self.api_key = api_key
+        self.team_handle = team_handle
+
+    def validate_api_key(self) -> bool:
+        try:
+            self._validate_user_and_team()
+            return True
+        except requests.HTTPError as e:
+            if e.response.status_code == 401:
+                raise_invalid_credentials_error(
+                    fields=[["creds", "api_key"]], details="Invalid API key"
+                )
+            elif e.response.status_code == 403:
+                raise_invalid_credentials_error(
+                    fields=[["creds", "api_key"]],
+                    details="Authenticated user does note have required permissions",
+                )
+            raise e
+        except ValueError as e:
+            error_message = str(e)
+            if "No Hot Aisle teams found" in error_message:
+                raise_invalid_credentials_error(
+                    fields=[["creds", "api_key"]],
+                    details="Valid API key but no teams found for this user",
+                )
+            elif "not found" in error_message:
+                raise_invalid_credentials_error(
+                    fields=[["team_handle"]], details=f"Team handle '{self.team_handle}' not found"
+                )
+            raise e
+
+    def _validate_user_and_team(self) -> None:
+        url = f"{API_URL}/user/"
+        response = self._make_request("GET", url)
+        response.raise_for_status()
+        user_data = response.json()
+
+        teams = user_data.get("teams", [])
+        if not teams:
+            raise ValueError("No Hot Aisle teams found for this user")
+
+        available_teams = [team["handle"] for team in teams]
+        if self.team_handle not in available_teams:
+            raise ValueError(f"Hot Aisle team '{self.team_handle}' not found.")
+
+    def upload_ssh_key(self, public_key: str) -> bool:
+        url = f"{API_URL}/user/ssh_keys/"
+        payload = {"authorized_key": public_key}
+
+        response = self._make_request("POST", url, json=payload)
+
+        if response.status_code == 409:
+            return True  # Key already exists - success
+        response.raise_for_status()
+        return True
+
+    def create_virtual_machine(self, vm_payload: Dict[str, Any]) -> Dict[str, Any]:
+        url = f"{API_URL}/teams/{self.team_handle}/virtual_machines/"
+        response = self._make_request("POST", url, json=vm_payload)
+        response.raise_for_status()
+        vm_data = response.json()
+        return vm_data
+
+    def get_vm_state(self, vm_name: str) -> str:
+        url = f"{API_URL}/teams/{self.team_handle}/virtual_machines/{vm_name}/state/"
+        response = self._make_request("GET", url)
+        response.raise_for_status()
+        state_data = response.json()
+        return state_data["state"]
+
+    def terminate_virtual_machine(self, vm_name: str) -> None:
+        url = f"{API_URL}/teams/{self.team_handle}/virtual_machines/{vm_name}/"
+        response = self._make_request("DELETE", url)
+        if response.status_code == 404:
+            logger.debug("Hot Aisle virtual machine %s not found", vm_name)
+            return
+        response.raise_for_status()
+
+    def _make_request(
+        self, method: str, url: str, json: Optional[Dict[str, Any]] = None, timeout: int = 30
+    ) -> requests.Response:
+        headers = {
+            "accept": "application/json",
+            "Authorization": f"Token {self.api_key}",
+        }
+        if json is not None:
+            headers["Content-Type"] = "application/json"
+
+        return requests.request(
+            method=method,
+            url=url,
+            headers=headers,
+            json=json,
+            timeout=timeout,
+        )

dstack/_internal/core/backends/hotaisle/backend.py

@@ -0,0 +1,16 @@
+from dstack._internal.core.backends.base.backend import Backend
+from dstack._internal.core.backends.hotaisle.compute import HotAisleCompute
+from dstack._internal.core.backends.hotaisle.models import HotAisleConfig
+from dstack._internal.core.models.backends.base import BackendType
+
+
+class HotAisleBackend(Backend):
+    TYPE = BackendType.HOTAISLE
+    COMPUTE_CLASS = HotAisleCompute
+
+    def __init__(self, config: HotAisleConfig):
+        self.config = config
+        self._compute = HotAisleCompute(self.config)
+
+    def compute(self) -> HotAisleCompute:
+        return self._compute

dstack/_internal/core/backends/hotaisle/compute.py

@@ -0,0 +1,225 @@
+import shlex
+import subprocess
+import tempfile
+from threading import Thread
+from typing import List, Optional
+
+import gpuhunt
+from gpuhunt.providers.hotaisle import HotAisleProvider
+
+from dstack._internal.core.backends.base.compute import (
+    Compute,
+    ComputeWithCreateInstanceSupport,
+    get_shim_commands,
+)
+from dstack._internal.core.backends.base.offers import get_catalog_offers
+from dstack._internal.core.backends.hotaisle.api_client import HotAisleAPIClient
+from dstack._internal.core.backends.hotaisle.models import HotAisleConfig
+from dstack._internal.core.models.backends.base import BackendType
+from dstack._internal.core.models.common import CoreModel
+from dstack._internal.core.models.instances import (
+    InstanceAvailability,
+    InstanceConfiguration,
+    InstanceOfferWithAvailability,
+)
+from dstack._internal.core.models.placement import PlacementGroup
+from dstack._internal.core.models.runs import JobProvisioningData, Requirements
+from dstack._internal.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+MAX_INSTANCE_NAME_LEN = 60
+
+
+INSTANCE_TYPE_SPECS = {
+    "1x MI300X 8x Xeon Platinum 8462Y+": {
+        "cpu_model": "Xeon Platinum 8462Y+",
+        "cpu_frequency": 2800000000,
+        "cpu_manufacturer": "Intel",
+    },
+    "1x MI300X 13x Xeon Platinum 8470": {
+        "cpu_model": "Xeon Platinum 8470",
+        "cpu_frequency": 2000000000,
+        "cpu_manufacturer": "Intel",
+    },
+}
+
+
+class HotAisleCompute(
+    ComputeWithCreateInstanceSupport,
+    Compute,
+):
+    def __init__(self, config: HotAisleConfig):
+        super().__init__()
+        self.config = config
+        self.api_client = HotAisleAPIClient(config.creds.api_key, config.team_handle)
+        self.catalog = gpuhunt.Catalog(balance_resources=False, auto_reload=False)
+        self.catalog.add_provider(
+            HotAisleProvider(api_key=config.creds.api_key, team_handle=config.team_handle)
+        )
+
+    def get_offers(
+        self, requirements: Optional[Requirements] = None
+    ) -> List[InstanceOfferWithAvailability]:
+        offers = get_catalog_offers(
+            backend=BackendType.HOTAISLE,
+            locations=self.config.regions or None,
+            requirements=requirements,
+            catalog=self.catalog,
+        )
+
+        supported_offers = []
+        for offer in offers:
+            if offer.instance.name in INSTANCE_TYPE_SPECS:
+                supported_offers.append(
+                    InstanceOfferWithAvailability(
+                        **offer.dict(), availability=InstanceAvailability.AVAILABLE
+                    )
+                )
+            else:
+                logger.warning(
+                    f"Skipping unsupported Hot Aisle instance type: {offer.instance.name}"
+                )
+
+        return supported_offers
+
+    def get_payload_from_offer(self, instance_type) -> dict:
+        instance_type_name = instance_type.name
+        cpu_specs = INSTANCE_TYPE_SPECS[instance_type_name]
+        cpu_cores = instance_type.resources.cpus
+
+        return {
+            "cpu_cores": cpu_cores,
+            "cpus": {
+                "count": 1,
+                "manufacturer": cpu_specs["cpu_manufacturer"],
+                "model": cpu_specs["cpu_model"],
+                "cores": cpu_cores,
+                "frequency": cpu_specs["cpu_frequency"],
+            },
+            "disk_capacity": instance_type.resources.disk.size_mib * 1024**2,
+            "ram_capacity": instance_type.resources.memory_mib * 1024**2,
+            "gpus": [
+                {
+                    "count": len(instance_type.resources.gpus),
+                    "manufacturer": instance_type.resources.gpus[0].vendor,
+                    "model": instance_type.resources.gpus[0].name,
+                }
+            ],
+        }
+
+    def create_instance(
+        self,
+        instance_offer: InstanceOfferWithAvailability,
+        instance_config: InstanceConfiguration,
+        placement_group: Optional[PlacementGroup],
+    ) -> JobProvisioningData:
+        project_ssh_key = instance_config.ssh_keys[0]
+        self.api_client.upload_ssh_key(project_ssh_key.public)
+        vm_payload = self.get_payload_from_offer(instance_offer.instance)
+        vm_data = self.api_client.create_virtual_machine(vm_payload)
+        return JobProvisioningData(
+            backend=instance_offer.backend,
+            instance_type=instance_offer.instance,
+            instance_id=vm_data["name"],
+            hostname=None,
+            internal_ip=None,
+            region=instance_offer.region,
+            price=instance_offer.price,
+            username="hotaisle",
+            ssh_port=22,
+            dockerized=True,
+            ssh_proxy=None,
+            backend_data=HotAisleInstanceBackendData(
+                ip_address=vm_data["ip_address"], vm_id=vm_data["name"]
+            ).json(),
+        )
+
+    def update_provisioning_data(
+        self,
+        provisioning_data: JobProvisioningData,
+        project_ssh_public_key: str,
+        project_ssh_private_key: str,
+    ):
+        vm_state = self.api_client.get_vm_state(provisioning_data.instance_id)
+        if vm_state == "running":
+            if provisioning_data.hostname is None and provisioning_data.backend_data:
+                backend_data = HotAisleInstanceBackendData.load(provisioning_data.backend_data)
+                provisioning_data.hostname = backend_data.ip_address
+            commands = get_shim_commands(
+                authorized_keys=[project_ssh_public_key],
+                arch=provisioning_data.instance_type.resources.cpu_arch,
+            )
+            launch_command = "sudo sh -c " + shlex.quote(" && ".join(commands))
+            thread = Thread(
+                target=_start_runner,
+                kwargs={
+                    "hostname": provisioning_data.hostname,
+                    "project_ssh_private_key": project_ssh_private_key,
+                    "launch_command": launch_command,
+                },
+                daemon=True,
+            )
+            thread.start()
+
+    def terminate_instance(
+        self, instance_id: str, region: str, backend_data: Optional[str] = None
+    ):
+        vm_name = instance_id
+        self.api_client.terminate_virtual_machine(vm_name)
+
+
+def _start_runner(
+    hostname: str,
+    project_ssh_private_key: str,
+    launch_command: str,
+):
+    _launch_runner(
+        hostname=hostname,
+        ssh_private_key=project_ssh_private_key,
+        launch_command=launch_command,
+    )
+
+
+def _launch_runner(
+    hostname: str,
+    ssh_private_key: str,
+    launch_command: str,
+):
+    daemonized_command = f"{launch_command.rstrip('&')} >/tmp/dstack-shim.log 2>&1 & disown"
+    _run_ssh_command(
+        hostname=hostname,
+        ssh_private_key=ssh_private_key,
+        command=daemonized_command,
+    )
+
+
+def _run_ssh_command(hostname: str, ssh_private_key: str, command: str):
+    with tempfile.NamedTemporaryFile("w+", 0o600) as f:
+        f.write(ssh_private_key)
+        f.flush()
+        subprocess.run(
+            [
+                "ssh",
+                "-F",
+                "none",
+                "-o",
+                "StrictHostKeyChecking=no",
+                "-i",
+                f.name,
+                f"hotaisle@{hostname}",
+                command,
+            ],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+
+
+class HotAisleInstanceBackendData(CoreModel):
+    ip_address: str
+    vm_id: Optional[str] = None
+
+    @classmethod
+    def load(cls, raw: Optional[str]) -> "HotAisleInstanceBackendData":
+        assert raw is not None
+        return cls.__response__.parse_raw(raw)