dstack 0.18.43__py3-none-any.whl → 0.18.44__py3-none-any.whl

dstack/_internal/server/services/metrics.py

@@ -1,113 +1,146 @@
 import json
+from collections import defaultdict
+from collections.abc import Sequence
 from datetime import datetime, timezone
+from typing import Optional
 
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from dstack._internal.core.errors import ResourceNotExistsError
 from dstack._internal.core.models.metrics import JobMetrics, Metric
-from dstack._internal.server.models import JobMetricsPoint, JobModel, ProjectModel
-from dstack._internal.server.services.jobs import get_run_job_model
+from dstack._internal.server.models import JobMetricsPoint, JobModel
+from dstack._internal.utils.logging import get_logger
 
-
-async def get_job_metrics(
-    session: AsyncSession,
-    project: ProjectModel,
-    run_name: str,
-    replica_num: int,
-    job_num: int,
-) -> JobMetrics:
-    job_model = await get_run_job_model(
-        session=session,
-        project=project,
-        run_name=run_name,
-        replica_num=replica_num,
-        job_num=job_num,
-    )
-    if job_model is None:
-        raise ResourceNotExistsError("Found no job with given parameters")
-    job_metrics = await _get_job_metrics(
-        session=session,
-        job_model=job_model,
-    )
-    return job_metrics
+logger = get_logger(__name__)
 
 
-async def _get_job_metrics(
+async def get_job_metrics(
     session: AsyncSession,
     job_model: JobModel,
+    after: Optional[datetime] = None,
+    before: Optional[datetime] = None,
+    limit: Optional[int] = None,
 ) -> JobMetrics:
-    res = await session.execute(
+    """
+    Returns metrics ordered from the latest to the earliest.
+
+    Expected usage:
+        * limit=100 — get the latest 100 points
+        * after=<now - 1 hour> — get points for the last one hour
+        * before=<earliest timestamp from the last batch>, limit=100 ­— paginate back in history
+    """
+    stmt = (
         select(JobMetricsPoint)
         .where(JobMetricsPoint.job_id == job_model.id)
         .order_by(JobMetricsPoint.timestamp_micro.desc())
-        .limit(2)
     )
+    if after is not None:
+        # we need +1 point for cpu_usage_percent, thus >=
+        stmt = stmt.where(JobMetricsPoint.timestamp_micro >= _datetime_to_unix_time_micro(after))
+    if before is not None:
+        stmt = stmt.where(JobMetricsPoint.timestamp_micro < _datetime_to_unix_time_micro(before))
+    if limit is not None:
+        # +1 for cpu_usage_percent
+        stmt = stmt.limit(limit + 1)
+    res = await session.execute(stmt)
     points = res.scalars().all()
+    # we need at least 2 points to calculate cpu_usage_percent
     if len(points) < 2:
         return JobMetrics(metrics=[])
-    last_point = points[0]
-    prev_point = points[1]
-    return _calculate_job_metrics(last_point, prev_point)
+    return _calculate_job_metrics(points)
+
+
+def _calculate_job_metrics(points: Sequence[JobMetricsPoint]) -> JobMetrics:
+    timestamps: list[datetime] = []
+    cpu_usage_points: list[int] = []
+    memory_usage_points: list[int] = []
+    memory_working_set_points: list[int] = []
+    gpus_memory_usage_points: defaultdict[int, list[int]] = defaultdict(list)
+    gpus_util_points: defaultdict[int, list[int]] = defaultdict(list)
 
+    gpus_detected_num: Optional[int] = None
+    gpus_detected_num_mismatch: bool = False
+    for point, prev_point in zip(points, points[1:]):
+        timestamps.append(_unix_time_micro_to_datetime(point.timestamp_micro))
+        cpu_usage_points.append(_get_cpu_usage(point, prev_point))
+        memory_usage_points.append(point.memory_usage_bytes)
+        memory_working_set_points.append(point.memory_working_set_bytes)
+        gpus_memory_usage = json.loads(point.gpus_memory_usage_bytes)
+        gpus_util = json.loads(point.gpus_util_percent)
+        if gpus_detected_num is None:
+            gpus_detected_num = len(gpus_memory_usage)
+        if len(gpus_memory_usage) != gpus_detected_num or len(gpus_util) != gpus_detected_num:
+            gpus_detected_num_mismatch = True
+        if not gpus_detected_num_mismatch:
+            for i in range(gpus_detected_num):
+                gpus_memory_usage_points[i].append(gpus_memory_usage[i])
+                gpus_util_points[i].append(gpus_util[i])
 
-def _calculate_job_metrics(last_point: JobMetricsPoint, prev_point: JobMetricsPoint) -> JobMetrics:
-    metrics = []
-    timestamp = _unix_time_micro_to_datetime(last_point.timestamp_micro)
-    metrics.append(
+    metrics: list[Metric] = [
         Metric(
             name="cpu_usage_percent",
-            timestamps=[timestamp],
-            values=[_get_cpu_usage(last_point, prev_point)],
-        )
-    )
-    metrics.append(
+            timestamps=timestamps,
+            values=cpu_usage_points,
+        ),
         Metric(
             name="memory_usage_bytes",
-            timestamps=[timestamp],
-            values=[last_point.memory_usage_bytes],
-        )
-    )
-    metrics.append(
+            timestamps=timestamps,
+            values=memory_usage_points,
+        ),
         Metric(
             name="memory_working_set_bytes",
-            timestamps=[timestamp],
-            values=[last_point.memory_working_set_bytes],
-        )
-    )
-
-    gpus_memory_usage_bytes = json.loads(last_point.gpus_memory_usage_bytes)
-    gpus_util_percent = json.loads(last_point.gpus_util_percent)
-    gpus_detected_num = len(gpus_memory_usage_bytes)
-    metrics.append(
-        Metric(
-            name="gpus_detected_num",
-            timestamps=[timestamp],
-            values=[gpus_detected_num],
-        )
-    )
-    for i in range(gpus_detected_num):
+            timestamps=timestamps,
+            values=memory_working_set_points,
+        ),
+    ]
+    if gpus_detected_num_mismatch:
+        # If number of GPUs changed in the time window, skip GPU metrics altogether, otherwise
+        # results can be unpredictable (e.g, one GPU takes place of another, as they are
+        # identified by an array index only).
+        logger.warning("gpus_detected_num mismatch, skipping GPU metrics")
+    else:
         metrics.append(
+            # As gpus_detected_num expected to be constant, we add only two points — the latest
+            # and the earliest in the batch
             Metric(
-                name=f"gpu_memory_usage_bytes_gpu{i}",
-                timestamps=[timestamp],
-                values=[gpus_memory_usage_bytes[i]],
+                name="gpus_detected_num",
+                timestamps=[timestamps[0], timestamps[-1]]
+                if len(timestamps) > 1
+                else [timestamps[0]],
+                values=[gpus_detected_num, gpus_detected_num]
+                if len(timestamps) > 1
+                else [gpus_detected_num],
             )
         )
-        metrics.append(
-            Metric(
-                name=f"gpu_util_percent_gpu{i}",
-                timestamps=[timestamp],
-                values=[gpus_util_percent[i]],
+        for index, gpu_memory_usage_points in gpus_memory_usage_points.items():
+            metrics.append(
+                Metric(
+                    name=f"gpu_memory_usage_bytes_gpu{index}",
+                    timestamps=timestamps,
+                    values=gpu_memory_usage_points,
+                )
+            )
+        for index, gpu_util_points in gpus_util_points.items():
+            metrics.append(
+                Metric(
+                    name=f"gpu_util_percent_gpu{index}",
+                    timestamps=timestamps,
+                    values=gpu_util_points,
+                )
             )
-        )
     return JobMetrics(metrics=metrics)
 
 
 def _get_cpu_usage(last_point: JobMetricsPoint, prev_point: JobMetricsPoint) -> int:
     window = last_point.timestamp_micro - prev_point.timestamp_micro
+    if window == 0:
+        return 0
     return round((last_point.cpu_usage_micro - prev_point.cpu_usage_micro) / window * 100)
 
 
 def _unix_time_micro_to_datetime(unix_time_ms: int) -> datetime:
     return datetime.fromtimestamp(unix_time_ms / 1_000_000, tz=timezone.utc)
+
+
+def _datetime_to_unix_time_micro(dt: datetime) -> int:
+    return int(dt.timestamp() * 1_000_000)

dstack/_internal/server/services/prometheus.py

@@ -0,0 +1,87 @@
+from collections.abc import Generator, Iterable
+
+from prometheus_client import Metric
+from prometheus_client.parser import text_string_to_metric_families
+from prometheus_client.samples import Sample
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import joinedload
+
+from dstack._internal.core.models.runs import JobStatus
+from dstack._internal.server.models import JobModel, JobPrometheusMetrics, ProjectModel
+
+
+async def get_metrics(session: AsyncSession) -> str:
+    res = await session.execute(
+        select(JobPrometheusMetrics)
+        .join(JobModel)
+        .join(ProjectModel)
+        .where(JobModel.status.in_([JobStatus.RUNNING]))
+        .order_by(ProjectModel.name, JobModel.job_name)
+        .options(joinedload(JobPrometheusMetrics.job).joinedload(JobModel.project))
+    )
+    metrics_models = res.scalars().all()
+    return _process_metrics(metrics_models)
+
+
+async def get_project_metrics(session: AsyncSession, project: ProjectModel) -> str:
+    res = await session.execute(
+        select(JobPrometheusMetrics)
+        .join(JobModel)
+        .where(
+            JobModel.project_id == project.id,
+            JobModel.status.in_([JobStatus.RUNNING]),
+        )
+        .order_by(JobModel.job_name)
+        .options(joinedload(JobPrometheusMetrics.job).joinedload(JobModel.project))
+    )
+    metrics_models = res.scalars().all()
+    return _process_metrics(metrics_models)
+
+
+def _process_metrics(metrics_models: Iterable[JobPrometheusMetrics]) -> str:
+    metrics = _parse_and_enrich_metrics(metrics_models)
+    if not metrics:
+        return ""
+    return "\n".join(_render_metrics(metrics)) + "\n"
+
+
+def _parse_and_enrich_metrics(metrics_models: Iterable[JobPrometheusMetrics]) -> list[Metric]:
+    metrics: dict[str, Metric] = {}
+    for metrics_model in metrics_models:
+        for metric in text_string_to_metric_families(metrics_model.text):
+            samples = metric.samples
+            metric.samples = []
+            name = metric.name
+            metric = metrics.setdefault(name, metric)
+            for sample in samples:
+                labels = sample.labels
+                labels.update(_get_dstack_labels(metrics_model.job))
+                # text_string_to_metric_families "fixes" counter names appending _total,
+                # we rebuild Sample to revert this
+                metric.samples.append(Sample(name, labels, *sample[2:]))
+    return list(metrics.values())
+
+
+def _get_dstack_labels(job: JobModel) -> dict[str, str]:
+    return {
+        "dstack_project_name": job.project.name,
+        "dstack_run_name": job.run_name,
+        "dstack_job_name": job.job_name,
+        "dstack_job_num": str(job.job_num),
+        "dstack_replica_num": str(job.replica_num),
+    }
+
+
+def _render_metrics(metrics: Iterable[Metric]) -> Generator[str, None, None]:
+    for metric in metrics:
+        yield f"# HELP {metric.name} {metric.documentation}"
+        yield f"# TYPE {metric.name} {metric.type}"
+        for sample in metric.samples:
+            parts: list[str] = [f"{sample.name}{{"]
+            parts.extend(",".join(f'{name}="{value}"' for name, value in sample.labels.items()))
+            parts.append(f"}} {sample.value}")
+            # text_string_to_metric_families converts milliseconds to float seconds
+            if isinstance(sample.timestamp, float):
+                parts.append(f" {int(sample.timestamp * 1000)}")
+            yield "".join(parts)

dstack/_internal/server/services/runner/client.py

@@ -178,9 +178,6 @@ class ShimClient:
     # API v1 (a.k.a. Legacy API) — `/api/{submit,pull,stop}`
     _API_V2_MIN_SHIM_VERSION = (0, 18, 34)
 
-    # A surrogate task ID for API-v1-over-v2 emulation (`_v2_compat_*` methods)
-    _LEGACY_TASK_ID = "00000000-0000-0000-0000-000000000000"
-
     _shim_version: Optional["_Version"]
     _api_version: int
     _negotiated: bool = False
@@ -339,6 +336,20 @@ class ShimClient:
         resp = self._request("GET", "/api/pull", raise_for_status=True)
         return self._response(LegacyPullResponse, resp)
 
+    # Metrics
+
+    def get_task_metrics(self, task_id: "_TaskID") -> Optional[str]:
+        resp = self._request("GET", f"/metrics/tasks/{task_id}")
+        if resp.status_code == HTTPStatus.NOT_FOUND:
+            # Metrics exporter is not installed or old shim version
+            return None
+        if resp.status_code == HTTPStatus.BAD_GATEWAY:
+            # Metrics exporter is not available or returned an error
+            logger.info("failed to collect metrics for task %s: %s", task_id, resp.text)
+            return None
+        self._raise_for_status(resp)
+        return resp.text
+
     # Private methods used for public methods implementations
 
     def _request(

dstack/_internal/server/services/runs.py

@@ -16,6 +16,7 @@ from dstack._internal.core.errors import (
     ServerClientError,
 )
 from dstack._internal.core.models.common import ApplyAction, is_core_model_instance
+from dstack._internal.core.models.configurations import AnyRunConfiguration
 from dstack._internal.core.models.instances import (
     InstanceAvailability,
     InstanceOfferWithAvailability,
@@ -47,6 +48,7 @@ from dstack._internal.core.models.volumes import (
 )
 from dstack._internal.core.services import validate_dstack_resource_name
 from dstack._internal.core.services.diff import diff_models
+from dstack._internal.server import settings
 from dstack._internal.server.db import get_db
 from dstack._internal.server.models import (
     JobModel,
@@ -552,6 +554,7 @@ async def stop_run(session: AsyncSession, run_model: RunModel, abort: bool):
     res = await session.execute(
         select(RunModel)
         .where(RunModel.id == run_model.id)
+        .order_by(RunModel.id)  # take locks in order
         .with_for_update()
         .execution_options(populate_existing=True)
     )
@@ -559,6 +562,7 @@ async def stop_run(session: AsyncSession, run_model: RunModel, abort: bool):
     await session.execute(
         select(JobModel)
         .where(JobModel.run_id == run_model.id)
+        .order_by(JobModel.id)  # take locks in order
         .with_for_update()
         .execution_options(populate_existing=True)
     )
@@ -592,7 +596,10 @@ async def delete_runs(
     await session.commit()
     async with get_locker().lock_ctx(RunModel.__tablename__, run_ids):
         res = await session.execute(
-            select(RunModel).where(RunModel.id.in_(run_ids)).with_for_update()
+            select(RunModel)
+            .where(RunModel.id.in_(run_ids))
+            .order_by(RunModel.id)  # take locks in order
+            .with_for_update()
         )
         run_models = res.scalars().all()
         active_runs = [r for r in run_models if not r.status.is_finished()]
@@ -832,12 +839,23 @@ def _validate_run_spec_and_set_defaults(run_spec: RunSpec):
         run_spec.repo_id = DEFAULT_VIRTUAL_REPO_ID
     if run_spec.repo_data is None:
         run_spec.repo_data = VirtualRunRepoData()
+    if (
+        run_spec.merged_profile.utilization_policy is not None
+        and run_spec.merged_profile.utilization_policy.time_window
+        > settings.SERVER_METRICS_TTL_SECONDS
+    ):
+        raise ServerClientError(
+            f"Maximum utilization_policy.time_window is {settings.SERVER_METRICS_TTL_SECONDS}s"
+        )
 
 
 _UPDATABLE_SPEC_FIELDS = ["repo_code_hash", "configuration"]
-# Most service fields can be updated via replica redeployment.
-# TODO: Allow updating other fields when a rolling deployment is supported.
-_UPDATABLE_CONFIGURATION_FIELDS = ["replicas", "scaling", "strip_prefix"]
+_CONF_TYPE_TO_UPDATABLE_FIELDS = {
+    "dev-environment": ["inactivity_duration"],
+    # Most service fields can be updated via replica redeployment.
+    # TODO: Allow updating other fields when rolling deployment is supported.
+    "service": ["replicas", "scaling", "strip_prefix"],
+}
 
 
 def _can_update_run_spec(current_run_spec: RunSpec, new_run_spec: RunSpec) -> bool:
@@ -850,11 +868,6 @@ def _can_update_run_spec(current_run_spec: RunSpec, new_run_spec: RunSpec) -> bo
 
 
 def _check_can_update_run_spec(current_run_spec: RunSpec, new_run_spec: RunSpec):
-    if (
-        current_run_spec.configuration.type != "service"
-        or new_run_spec.configuration.type != "service"
-    ):
-        raise ServerClientError("Can only update service run configuration")
     spec_diff = diff_models(current_run_spec, new_run_spec)
     changed_spec_fields = list(spec_diff.keys())
     for key in changed_spec_fields:
@@ -863,13 +876,28 @@ def _check_can_update_run_spec(current_run_spec: RunSpec, new_run_spec: RunSpec)
                 f"Failed to update fields {changed_spec_fields}."
                 f" Can only update {_UPDATABLE_SPEC_FIELDS}."
             )
-    configuration_diff = diff_models(current_run_spec.configuration, new_run_spec.configuration)
-    changed_configuration_fields = list(configuration_diff.keys())
-    for key in changed_configuration_fields:
-        if key not in _UPDATABLE_CONFIGURATION_FIELDS:
+    _check_can_update_configuration(current_run_spec.configuration, new_run_spec.configuration)
+
+
+def _check_can_update_configuration(
+    current: AnyRunConfiguration, new: AnyRunConfiguration
+) -> None:
+    if current.type != new.type:
+        raise ServerClientError(
+            f"Configuration type changed from {current.type} to {new.type}, cannot update"
+        )
+    updatable_fields = _CONF_TYPE_TO_UPDATABLE_FIELDS.get(new.type)
+    if updatable_fields is None:
+        raise ServerClientError(
+            f"Can only update {', '.join(_CONF_TYPE_TO_UPDATABLE_FIELDS)} configurations."
+            f" Not {new.type}"
+        )
+    diff = diff_models(current, new)
+    changed_fields = list(diff.keys())
+    for key in changed_fields:
+        if key not in updatable_fields:
             raise ServerClientError(
-                f"Failed to update fields {changed_configuration_fields}."
-                f" Can only update {_UPDATABLE_CONFIGURATION_FIELDS}"
+                f"Failed to update fields {changed_fields}. Can only update {updatable_fields}"
             )
 
 

dstack/_internal/server/services/volumes.py

@@ -264,6 +264,7 @@ async def delete_volumes(session: AsyncSession, project: ProjectModel, names: Li
             .options(selectinload(VolumeModel.user))
             .options(selectinload(VolumeModel.attachments))
             .execution_options(populate_existing=True)
+            .order_by(VolumeModel.id)  # take locks in order
             .with_for_update()
         )
         volume_models = res.scalars().unique().all()

dstack/_internal/server/settings.py

@@ -31,6 +31,8 @@ ALEMBIC_MIGRATIONS_LOCATION = os.getenv(
 DB_POOL_SIZE = int(os.getenv("DSTACK_DB_POOL_SIZE", 10))
 DB_MAX_OVERFLOW = int(os.getenv("DSTACK_DB_MAX_OVERFLOW", 10))
 
+MAX_OFFERS_TRIED = int(os.getenv("DSTACK_SERVER_MAX_OFFERS_TRIED", 25))
+
 SERVER_CONFIG_DISABLED = os.getenv("DSTACK_SERVER_CONFIG_DISABLED") is not None
 SERVER_CONFIG_ENABLED = not SERVER_CONFIG_DISABLED
 
@@ -74,3 +76,4 @@ LOCAL_BACKEND_ENABLED = os.getenv("DSTACK_LOCAL_BACKEND_ENABLED") is not None
 UPDATE_DEFAULT_PROJECT = os.getenv("DSTACK_UPDATE_DEFAULT_PROJECT") is not None
 DO_NOT_UPDATE_DEFAULT_PROJECT = os.getenv("DSTACK_DO_NOT_UPDATE_DEFAULT_PROJECT") is not None
 SKIP_GATEWAY_UPDATE = os.getenv("DSTACK_SKIP_GATEWAY_UPDATE", None) is not None
+ENABLE_PROMETHEUS_METRICS = os.getenv("DSTACK_ENABLE_PROMETHEUS_METRICS", None) is not None

dstack/_internal/server/statics/index.html

@@ -1,3 +1,3 @@
 <!doctype html><html lang="en"><head><meta charset="utf-8"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><title>dstack</title><meta name="description" content="Get GPUs at the best prices and availability from a wide range of providers. No cloud account of your own is required.
 "/><link rel="preconnect" href="https://fonts.googleapis.com"><link rel="preconnect" href="https://fonts.gstatic.com" crossorigin><link href="https://fonts.googleapis.com/css2?family=Roboto:ital,wght@0,100;0,300;0,400;0,500;0,700;0,900;1,100;1,300;1,400;1,500;1,700;1,900&display=swap" rel="stylesheet"><meta name="og:title" content="dstack"><meta name="og:type" content="article"><meta name="og:image" content="/splash_thumbnail.png"><meta name="og:description" content="Get GPUs at the best prices and availability from a wide range of providers. No cloud account of your own is required.
-"><link rel="icon" type="image/x-icon" href="/assets/favicon.ico"><link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png"><link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png"><link rel="icon" type="image/png" sizes="48x48" href="/assets/favicon-48x48.png"><link rel="manifest" href="/assets/manifest.webmanifest"><meta name="mobile-web-app-capable" content="yes"><meta name="theme-color" content="#fff"><meta name="application-name" content="dstackai"><link rel="apple-touch-icon" sizes="57x57" href="/assets/apple-touch-icon-57x57.png"><link rel="apple-touch-icon" sizes="60x60" href="/assets/apple-touch-icon-60x60.png"><link rel="apple-touch-icon" sizes="72x72" href="/assets/apple-touch-icon-72x72.png"><link rel="apple-touch-icon" sizes="76x76" href="/assets/apple-touch-icon-76x76.png"><link rel="apple-touch-icon" sizes="114x114" href="/assets/apple-touch-icon-114x114.png"><link rel="apple-touch-icon" sizes="120x120" href="/assets/apple-touch-icon-120x120.png"><link rel="apple-touch-icon" sizes="144x144" href="/assets/apple-touch-icon-144x144.png"><link rel="apple-touch-icon" sizes="152x152" href="/assets/apple-touch-icon-152x152.png"><link rel="apple-touch-icon" sizes="167x167" href="/assets/apple-touch-icon-167x167.png"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon-180x180.png"><link rel="apple-touch-icon" sizes="1024x1024" href="/assets/apple-touch-icon-1024x1024.png"><meta name="apple-mobile-web-app-capable" content="yes"><meta name="apple-mobile-web-app-status-bar-style" content="black-translucent"><meta name="apple-mobile-web-app-title" content="dstackai"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-640x1136.png"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1136x640.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-750x1334.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1334x750.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1125x2436.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2436x1125.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1170x2532.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2532x1170.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1179x2556.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2556x1179.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-828x1792.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1792x828.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2688.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2688x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2208.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2208x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1284x2778.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2778x1284.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1290x2796.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2796x1290.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1488x2266.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2266x1488.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1536x2048.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2048x1536.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1620x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1620.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1640x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1640.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2388.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2388x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2224.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2224x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-2048x2732.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2732x2048.png"><meta name="msapplication-TileColor" content="#fff"><meta name="msapplication-TileImage" content="/assets/mstile-144x144.png"><meta name="msapplication-config" content="/assets/browserconfig.xml"><link rel="yandex-tableau-widget" href="/assets/yandex-browser-manifest.json"><script defer="defer" src="/main-fe8fd9db55df8d10e648.js"></script><link href="/main-7510e71dfa9749a4e70e.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div class="b-page-header" id="header"></div><div id="root"></div></body></html>
+"><link rel="icon" type="image/x-icon" href="/assets/favicon.ico"><link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png"><link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png"><link rel="icon" type="image/png" sizes="48x48" href="/assets/favicon-48x48.png"><link rel="manifest" href="/assets/manifest.webmanifest"><meta name="mobile-web-app-capable" content="yes"><meta name="theme-color" content="#fff"><meta name="application-name" content="dstackai"><link rel="apple-touch-icon" sizes="57x57" href="/assets/apple-touch-icon-57x57.png"><link rel="apple-touch-icon" sizes="60x60" href="/assets/apple-touch-icon-60x60.png"><link rel="apple-touch-icon" sizes="72x72" href="/assets/apple-touch-icon-72x72.png"><link rel="apple-touch-icon" sizes="76x76" href="/assets/apple-touch-icon-76x76.png"><link rel="apple-touch-icon" sizes="114x114" href="/assets/apple-touch-icon-114x114.png"><link rel="apple-touch-icon" sizes="120x120" href="/assets/apple-touch-icon-120x120.png"><link rel="apple-touch-icon" sizes="144x144" href="/assets/apple-touch-icon-144x144.png"><link rel="apple-touch-icon" sizes="152x152" href="/assets/apple-touch-icon-152x152.png"><link rel="apple-touch-icon" sizes="167x167" href="/assets/apple-touch-icon-167x167.png"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon-180x180.png"><link rel="apple-touch-icon" sizes="1024x1024" href="/assets/apple-touch-icon-1024x1024.png"><meta name="apple-mobile-web-app-capable" content="yes"><meta name="apple-mobile-web-app-status-bar-style" content="black-translucent"><meta name="apple-mobile-web-app-title" content="dstackai"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-640x1136.png"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1136x640.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-750x1334.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1334x750.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1125x2436.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2436x1125.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1170x2532.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2532x1170.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1179x2556.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2556x1179.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-828x1792.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1792x828.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2688.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2688x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2208.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2208x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1284x2778.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2778x1284.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1290x2796.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2796x1290.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1488x2266.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2266x1488.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1536x2048.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2048x1536.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1620x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1620.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1640x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1640.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2388.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2388x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2224.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2224x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-2048x2732.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2732x2048.png"><meta name="msapplication-TileColor" content="#fff"><meta name="msapplication-TileImage" content="/assets/mstile-144x144.png"><meta name="msapplication-config" content="/assets/browserconfig.xml"><link rel="yandex-tableau-widget" href="/assets/yandex-browser-manifest.json"><script defer="defer" src="/main-4eb116b97819badd1e2c.js"></script><link href="/main-da9f8c06a69c20dac23e.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div class="b-page-header" id="header"></div><div id="root"></div></body></html>