dstack 0.19.0rc1__tar.gz → 0.19.1__tar.gz

{dstack-0.19.0rc1 → dstack-0.19.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dstack
-Version: 0.19.0rc1
+Version: 0.19.1
 Summary: dstack is an open-source orchestration engine for running AI workloads on any cloud or on-premises.
 Home-page: https://dstack.ai
 Author: Andrey Cheptsov

{dstack-0.19.0rc1 → dstack-0.19.1}/src/dstack/_internal/server/background/tasks/process_instances.py

@@ -64,6 +64,7 @@ from dstack._internal.core.models.runs import (
     Retry,
 )
 from dstack._internal.core.services.profiles import get_retry
+from dstack._internal.server import settings as server_settings
 from dstack._internal.server.background.tasks.common import get_provisioning_timeout
 from dstack._internal.server.db import get_session_ctx
 from dstack._internal.server.models import (
@@ -529,7 +530,9 @@ async def _create_instance(session: AsyncSession, instance: InstanceModel) -> No
             session=session, fleet_id=instance.fleet_id
         )
 
-    for backend, instance_offer in offers:
+    # Limit number of offers tried to prevent long-running processing
+    # in case all offers fail.
+    for backend, instance_offer in offers[: server_settings.MAX_OFFERS_TRIED]:
         if instance_offer.backend not in BACKENDS_WITH_CREATE_INSTANCE_SUPPORT:
             continue
         compute = backend.compute()
@@ -578,8 +581,13 @@ async def _create_instance(session: AsyncSession, instance: InstanceModel) -> No
                 extra={"instance_name": instance.name},
             )
             continue
-        except NotImplementedError:
-            # skip a backend without create_instance support, continue with next backend and offer
+        except Exception:
+            logger.exception(
+                "Got exception when launching %s in %s/%s",
+                instance_offer.instance.name,
+                instance_offer.backend.value,
+                instance_offer.region,
+            )
             continue
 
         instance.status = InstanceStatus.PROVISIONING
@@ -607,10 +615,11 @@ async def _create_instance(session: AsyncSession, instance: InstanceModel) -> No
 
     if not should_retry:
         instance.status = InstanceStatus.TERMINATED
-        instance.termination_reason = "No offers found"
+        instance.termination_reason = "All offers failed" if offers else "No offers found"
         logger.info(
-            "No offers found. Terminated instance %s",
+            "Terminated instance %s: %s",
             instance.name,
+            instance.termination_reason,
             extra={
                 "instance_name": instance.name,
                 "instance_status": InstanceStatus.TERMINATED.value,

{dstack-0.19.0rc1 → dstack-0.19.1}/src/dstack/_internal/server/routers/prometheus.py

@@ -6,8 +6,6 @@ from sqlalchemy.ext.asyncio import AsyncSession
 
 from dstack._internal.server import settings
 from dstack._internal.server.db import get_session
-from dstack._internal.server.deps import Project
-from dstack._internal.server.models import ProjectModel
 from dstack._internal.server.services import prometheus
 from dstack._internal.server.utils.routers import error_not_found
 
@@ -24,13 +22,3 @@ async def get_prometheus_metrics(
     if not settings.ENABLE_PROMETHEUS_METRICS:
         raise error_not_found()
     return await prometheus.get_metrics(session=session)
-
-
-@router.get("/metrics/project/{project_name}", deprecated=True)
-async def get_project_prometheus_metrics(
-    session: Annotated[AsyncSession, Depends(get_session)],
-    project: Annotated[ProjectModel, Depends(Project())],
-) -> str:
-    if not settings.ENABLE_PROMETHEUS_METRICS:
-        raise error_not_found()
-    return await prometheus.get_project_metrics(session=session, project=project)

{dstack-0.19.0rc1 → dstack-0.19.1}/src/dstack/_internal/server/services/jobs/configurators/extensions/cursor.py

@@ -35,7 +35,7 @@ class CursorDesktop:
 
     def get_print_readme_commands(self) -> List[str]:
         return [
-            "echo To open in VS Code Desktop, use link below:",
+            "echo To open in Cursor, use link below:",
             "echo ''",
             f"echo '  cursor://vscode-remote/ssh-remote+{self.run_name}/workflow'",  # TODO use $REPO_DIR
             "echo ''",

dstack-0.19.1/src/dstack/_internal/server/services/prometheus.py

@@ -0,0 +1,308 @@
+import itertools
+from collections import defaultdict
+from collections.abc import Generator, Iterable
+from datetime import timezone
+from typing import ClassVar
+from uuid import UUID
+
+from prometheus_client import Metric
+from prometheus_client.parser import text_string_to_metric_families
+from prometheus_client.samples import Sample
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import aliased, joinedload
+
+from dstack._internal.core.models.instances import InstanceStatus
+from dstack._internal.core.models.runs import JobStatus, RunSpec, RunStatus
+from dstack._internal.server.models import (
+    InstanceModel,
+    JobMetricsPoint,
+    JobModel,
+    JobPrometheusMetrics,
+    ProjectModel,
+    RunModel,
+    UserModel,
+)
+from dstack._internal.server.services.instances import get_instance_offer
+from dstack._internal.server.services.jobs import get_job_provisioning_data, get_job_runtime_data
+from dstack._internal.utils.common import get_current_datetime
+
+
+async def get_metrics(session: AsyncSession) -> str:
+    metrics_iter = itertools.chain(
+        await get_instance_metrics(session),
+        await get_run_metrics(session),
+        await get_job_metrics(session),
+    )
+    return "\n".join(_render_metrics(metrics_iter)) + "\n"
+
+
+async def get_instance_metrics(session: AsyncSession) -> Iterable[Metric]:
+    res = await session.execute(
+        select(InstanceModel)
+        .join(ProjectModel)
+        .where(
+            InstanceModel.deleted == False,
+            InstanceModel.status.in_(
+                [
+                    InstanceStatus.PROVISIONING,
+                    InstanceStatus.IDLE,
+                    InstanceStatus.BUSY,
+                    InstanceStatus.TERMINATING,
+                ]
+            ),
+        )
+        .order_by(ProjectModel.name, InstanceModel.name)
+        .options(
+            joinedload(InstanceModel.project),
+            joinedload(InstanceModel.fleet),
+        )
+    )
+    instances = res.unique().scalars().all()
+    metrics = _InstanceMetrics()
+    now = get_current_datetime()
+    for instance in instances:
+        fleet = instance.fleet
+        offer = get_instance_offer(instance)
+        gpu = ""
+        gpu_count = 0
+        if offer is not None and len(offer.instance.resources.gpus) > 0:
+            gpu = offer.instance.resources.gpus[0].name
+            gpu_count = len(offer.instance.resources.gpus)
+        labels: dict[str, str] = {
+            "dstack_project_name": instance.project.name,
+            "dstack_fleet_name": fleet.name if fleet is not None else "",
+            "dstack_fleet_id": str(fleet.id) if fleet is not None else "",
+            "dstack_instance_name": str(instance.name),
+            "dstack_instance_id": str(instance.id),
+            "dstack_instance_type": offer.instance.name if offer is not None else "",
+            "dstack_backend": instance.backend.value if instance.backend is not None else "",
+            "dstack_gpu": gpu,
+        }
+        duration = (now - instance.created_at.replace(tzinfo=timezone.utc)).total_seconds()
+        metrics.add_sample(_INSTANCE_DURATION, labels, duration)
+        metrics.add_sample(_INSTANCE_PRICE, labels, instance.price or 0.0)
+        metrics.add_sample(_INSTANCE_GPU_COUNT, labels, gpu_count)
+    return metrics.values()
+
+
+async def get_run_metrics(session: AsyncSession) -> Iterable[Metric]:
+    res = await session.execute(
+        select(ProjectModel.name, UserModel.name, RunModel.status, func.count(RunModel.id))
+        .join_from(RunModel, ProjectModel)
+        .join_from(RunModel, UserModel, RunModel.user_id == UserModel.id)
+        .group_by(ProjectModel.name, UserModel.name, RunModel.status)
+        .order_by(ProjectModel.name, UserModel.name, RunModel.status)
+    )
+    projects: dict[str, dict[str, dict[RunStatus, int]]] = defaultdict(
+        lambda: defaultdict(lambda: defaultdict(int))
+    )
+    for project_name, user_name, status, count in res.all():
+        projects[project_name][user_name][status] = count
+    metrics = _RunMetrics()
+    for project_name, users in projects.items():
+        for user_name, statuses in users.items():
+            labels: dict[str, str] = {
+                "dstack_project_name": project_name,
+                "dstack_user_name": user_name,
+            }
+            metrics.add_sample(_RUN_COUNT_TOTAL, labels, sum(statuses.values()))
+            metrics.add_sample(_RUN_COUNT_TERMINATED, labels, statuses[RunStatus.TERMINATED])
+            metrics.add_sample(_RUN_COUNT_FAILED, labels, statuses[RunStatus.FAILED])
+            metrics.add_sample(_RUN_COUNT_DONE, labels, statuses[RunStatus.DONE])
+    return metrics.values()
+
+
+async def get_job_metrics(session: AsyncSession) -> Iterable[Metric]:
+    res = await session.execute(
+        select(JobModel)
+        .join(ProjectModel)
+        .where(
+            JobModel.status.in_(
+                [
+                    JobStatus.PROVISIONING,
+                    JobStatus.PULLING,
+                    JobStatus.RUNNING,
+                    JobStatus.TERMINATING,
+                ]
+            )
+        )
+        .order_by(ProjectModel.name, JobModel.job_name)
+        .options(
+            joinedload(JobModel.project),
+            joinedload(JobModel.run).joinedload(RunModel.user),
+        )
+    )
+    jobs = res.scalars().all()
+    job_ids = {job.id for job in jobs}
+    job_metrics_points = await _get_job_metrics_points(session, job_ids)
+    job_prometheus_metrics = await _get_job_prometheus_metrics(session, job_ids)
+
+    metrics = _JobMetrics()
+    now = get_current_datetime()
+    for job in jobs:
+        jpd = get_job_provisioning_data(job)
+        if jpd is None:
+            continue
+        jrd = get_job_runtime_data(job)
+        resources = jpd.instance_type.resources
+        price = jpd.price
+        if jrd is not None and jrd.offer is not None:
+            resources = jrd.offer.instance.resources
+            price = jrd.offer.price
+        gpus = resources.gpus
+        cpus = resources.cpus
+        run_spec = RunSpec.__response__.parse_raw(job.run.run_spec)
+        labels = {
+            "dstack_project_name": job.project.name,
+            "dstack_user_name": job.run.user.name,
+            "dstack_run_name": job.run_name,
+            "dstack_run_id": str(job.run_id),
+            "dstack_job_name": job.job_name,
+            "dstack_job_id": str(job.id),
+            "dstack_job_num": str(job.job_num),
+            "dstack_replica_num": str(job.replica_num),
+            "dstack_run_type": run_spec.configuration.type,
+            "dstack_backend": jpd.get_base_backend().value,
+            "dstack_gpu": gpus[0].name if gpus else "",
+        }
+        duration = (now - job.submitted_at.replace(tzinfo=timezone.utc)).total_seconds()
+        metrics.add_sample(_JOB_DURATION, labels, duration)
+        metrics.add_sample(_JOB_PRICE, labels, price)
+        metrics.add_sample(_JOB_GPU_COUNT, labels, len(gpus))
+        metrics.add_sample(_JOB_CPU_COUNT, labels, cpus)
+        metrics.add_sample(_JOB_MEMORY_TOTAL, labels, resources.memory_mib * 1024 * 1024)
+        jmp = job_metrics_points.get(job.id)
+        if jmp is not None:
+            metrics.add_sample(_JOB_CPU_TIME, labels, jmp.cpu_usage_micro / 1_000_000)
+            metrics.add_sample(_JOB_MEMORY_USAGE, labels, jmp.memory_usage_bytes)
+            metrics.add_sample(_JOB_MEMORY_WORKING_SET, labels, jmp.memory_working_set_bytes)
+        jpm = job_prometheus_metrics.get(job.id)
+        if jpm is not None:
+            for metric in text_string_to_metric_families(jpm.text):
+                metrics.add_metric(metric, labels)
+    return metrics.values()
+
+
+_COUNTER = "counter"
+_GAUGE = "gauge"
+
+_INSTANCE_DURATION = "dstack_instance_duration_seconds_total"
+_INSTANCE_PRICE = "dstack_instance_price_dollars_per_hour"
+_INSTANCE_GPU_COUNT = "dstack_instance_gpu_count"
+_RUN_COUNT_TOTAL = "dstack_run_count_total"
+_RUN_COUNT_TERMINATED = "dstack_run_count_terminated_total"
+_RUN_COUNT_FAILED = "dstack_run_count_failed_total"
+_RUN_COUNT_DONE = "dstack_run_count_done_total"
+_JOB_DURATION = "dstack_job_duration_seconds_total"
+_JOB_PRICE = "dstack_job_price_dollars_per_hour"
+_JOB_GPU_COUNT = "dstack_job_gpu_count"
+_JOB_CPU_COUNT = "dstack_job_cpu_count"
+_JOB_CPU_TIME = "dstack_job_cpu_time_seconds_total"
+_JOB_MEMORY_TOTAL = "dstack_job_memory_total_bytes"
+_JOB_MEMORY_USAGE = "dstack_job_memory_usage_bytes"
+_JOB_MEMORY_WORKING_SET = "dstack_job_memory_working_set_bytes"
+
+
+class _Metrics(dict[str, Metric]):
+    metrics: ClassVar[list[tuple[str, str, str]]]
+
+    def __init__(self):
+        super().__init__()
+        for name, typ, documentation in self.metrics:
+            self[name] = Metric(name=name, documentation=documentation, typ=typ)
+
+    def add_sample(self, name: str, labels: dict[str, str], value: float) -> None:
+        # NOTE: Keeps reference to labels.
+        self[name].add_sample(name=name, labels=labels, value=value)
+
+    def add_metric(self, metric: Metric, labels: dict[str, str]) -> None:
+        # NOTE: Modifies and keeps reference to metric.
+        name = metric.name
+        samples = metric.samples
+        stored_metric = self.get(name)
+        if stored_metric is None:
+            stored_metric = metric
+            stored_metric.samples = []
+            self[name] = stored_metric
+        for sample in samples:
+            sample.labels.update(labels)
+            # text_string_to_metric_families "fixes" counter names appending _total,
+            # we rebuild Sample to revert this
+            stored_metric.samples.append(Sample(name, *sample[1:]))
+
+
+class _InstanceMetrics(_Metrics):
+    metrics = [
+        (_INSTANCE_DURATION, _COUNTER, "Total seconds the instance is running"),
+        (_INSTANCE_PRICE, _GAUGE, "Instance price, USD/hour"),
+        (_INSTANCE_GPU_COUNT, _GAUGE, "Instance GPU count"),
+    ]
+
+
+class _RunMetrics(_Metrics):
+    metrics = [
+        (_RUN_COUNT_TOTAL, _COUNTER, "Total runs count"),
+        (_RUN_COUNT_TERMINATED, _COUNTER, "Terminated runs count"),
+        (_RUN_COUNT_FAILED, _COUNTER, "Failed runs count"),
+        (_RUN_COUNT_DONE, _COUNTER, "Done runs count"),
+    ]
+
+
+class _JobMetrics(_Metrics):
+    metrics = [
+        (_JOB_DURATION, _COUNTER, "Total seconds the job is running"),
+        (_JOB_PRICE, _GAUGE, "Job instance price, USD/hour"),
+        (_JOB_GPU_COUNT, _GAUGE, "Job GPU count"),
+        (_JOB_CPU_COUNT, _GAUGE, "Job CPU count"),
+        (_JOB_CPU_TIME, _COUNTER, "Total CPU time consumed by the job, seconds"),
+        (_JOB_MEMORY_TOTAL, _GAUGE, "Total memory allocated for the job, bytes"),
+        (_JOB_MEMORY_USAGE, _GAUGE, "Memory used by the job (including cache), bytes"),
+        (_JOB_MEMORY_WORKING_SET, _GAUGE, "Memory used by the job (not including cache), bytes"),
+    ]
+
+
+async def _get_job_metrics_points(
+    session: AsyncSession, job_ids: Iterable[UUID]
+) -> dict[UUID, JobMetricsPoint]:
+    subquery = select(
+        JobMetricsPoint,
+        func.row_number()
+        .over(
+            partition_by=JobMetricsPoint.job_id,
+            order_by=JobMetricsPoint.timestamp_micro.desc(),
+        )
+        .label("row_number"),
+    ).subquery()
+    res = await session.execute(
+        select(aliased(JobMetricsPoint, subquery)).where(
+            subquery.c.row_number == 1,
+            subquery.c.job_id.in_(job_ids),
+        )
+    )
+    return {p.job_id: p for p in res.scalars().all()}
+
+
+async def _get_job_prometheus_metrics(
+    session: AsyncSession, job_ids: Iterable[UUID]
+) -> dict[UUID, JobPrometheusMetrics]:
+    res = await session.execute(
+        select(JobPrometheusMetrics).where(JobPrometheusMetrics.job_id.in_(job_ids))
+    )
+    return {p.job_id: p for p in res.scalars().all()}
+
+
+def _render_metrics(metrics: Iterable[Metric]) -> Generator[str, None, None]:
+    for metric in metrics:
+        if not metric.samples:
+            continue
+        yield f"# HELP {metric.name} {metric.documentation}"
+        yield f"# TYPE {metric.name} {metric.type}"
+        for sample in metric.samples:
+            parts: list[str] = [f"{sample.name}{{"]
+            parts.extend(",".join(f'{name}="{value}"' for name, value in sample.labels.items()))
+            parts.append(f"}} {float(sample.value)}")
+            # text_string_to_metric_families converts milliseconds to float seconds
+            if isinstance(sample.timestamp, float):
+                parts.append(f" {int(sample.timestamp * 1000)}")
+            yield "".join(parts)

{dstack-0.19.0rc1 → dstack-0.19.1}/src/dstack/_internal/server/statics/index.html

@@ -1,3 +1,3 @@
 <!doctype html><html lang="en"><head><meta charset="utf-8"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><title>dstack</title><meta name="description" content="Get GPUs at the best prices and availability from a wide range of providers. No cloud account of your own is required.
 "/><link rel="preconnect" href="https://fonts.googleapis.com"><link rel="preconnect" href="https://fonts.gstatic.com" crossorigin><link href="https://fonts.googleapis.com/css2?family=Roboto:ital,wght@0,100;0,300;0,400;0,500;0,700;0,900;1,100;1,300;1,400;1,500;1,700;1,900&display=swap" rel="stylesheet"><meta name="og:title" content="dstack"><meta name="og:type" content="article"><meta name="og:image" content="/splash_thumbnail.png"><meta name="og:description" content="Get GPUs at the best prices and availability from a wide range of providers. No cloud account of your own is required.
-"><link rel="icon" type="image/x-icon" href="/assets/favicon.ico"><link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png"><link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png"><link rel="icon" type="image/png" sizes="48x48" href="/assets/favicon-48x48.png"><link rel="manifest" href="/assets/manifest.webmanifest"><meta name="mobile-web-app-capable" content="yes"><meta name="theme-color" content="#fff"><meta name="application-name" content="dstackai"><link rel="apple-touch-icon" sizes="57x57" href="/assets/apple-touch-icon-57x57.png"><link rel="apple-touch-icon" sizes="60x60" href="/assets/apple-touch-icon-60x60.png"><link rel="apple-touch-icon" sizes="72x72" href="/assets/apple-touch-icon-72x72.png"><link rel="apple-touch-icon" sizes="76x76" href="/assets/apple-touch-icon-76x76.png"><link rel="apple-touch-icon" sizes="114x114" href="/assets/apple-touch-icon-114x114.png"><link rel="apple-touch-icon" sizes="120x120" href="/assets/apple-touch-icon-120x120.png"><link rel="apple-touch-icon" sizes="144x144" href="/assets/apple-touch-icon-144x144.png"><link rel="apple-touch-icon" sizes="152x152" href="/assets/apple-touch-icon-152x152.png"><link rel="apple-touch-icon" sizes="167x167" href="/assets/apple-touch-icon-167x167.png"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon-180x180.png"><link rel="apple-touch-icon" sizes="1024x1024" href="/assets/apple-touch-icon-1024x1024.png"><meta name="apple-mobile-web-app-capable" content="yes"><meta name="apple-mobile-web-app-status-bar-style" content="black-translucent"><meta name="apple-mobile-web-app-title" content="dstackai"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-640x1136.png"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1136x640.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-750x1334.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1334x750.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1125x2436.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2436x1125.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1170x2532.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2532x1170.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1179x2556.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2556x1179.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-828x1792.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1792x828.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2688.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2688x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2208.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2208x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1284x2778.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2778x1284.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1290x2796.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2796x1290.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1488x2266.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2266x1488.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1536x2048.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2048x1536.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1620x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1620.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1640x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1640.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2388.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2388x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2224.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2224x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-2048x2732.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2732x2048.png"><meta name="msapplication-TileColor" content="#fff"><meta name="msapplication-TileImage" content="/assets/mstile-144x144.png"><meta name="msapplication-config" content="/assets/browserconfig.xml"><link rel="yandex-tableau-widget" href="/assets/yandex-browser-manifest.json"><script defer="defer" src="/main-4fd5a4770eff59325ee3.js"></script><link href="/main-da9f8c06a69c20dac23e.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div class="b-page-header" id="header"></div><div id="root"></div></body></html>
+"><link rel="icon" type="image/x-icon" href="/assets/favicon.ico"><link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png"><link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png"><link rel="icon" type="image/png" sizes="48x48" href="/assets/favicon-48x48.png"><link rel="manifest" href="/assets/manifest.webmanifest"><meta name="mobile-web-app-capable" content="yes"><meta name="theme-color" content="#fff"><meta name="application-name" content="dstackai"><link rel="apple-touch-icon" sizes="57x57" href="/assets/apple-touch-icon-57x57.png"><link rel="apple-touch-icon" sizes="60x60" href="/assets/apple-touch-icon-60x60.png"><link rel="apple-touch-icon" sizes="72x72" href="/assets/apple-touch-icon-72x72.png"><link rel="apple-touch-icon" sizes="76x76" href="/assets/apple-touch-icon-76x76.png"><link rel="apple-touch-icon" sizes="114x114" href="/assets/apple-touch-icon-114x114.png"><link rel="apple-touch-icon" sizes="120x120" href="/assets/apple-touch-icon-120x120.png"><link rel="apple-touch-icon" sizes="144x144" href="/assets/apple-touch-icon-144x144.png"><link rel="apple-touch-icon" sizes="152x152" href="/assets/apple-touch-icon-152x152.png"><link rel="apple-touch-icon" sizes="167x167" href="/assets/apple-touch-icon-167x167.png"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon-180x180.png"><link rel="apple-touch-icon" sizes="1024x1024" href="/assets/apple-touch-icon-1024x1024.png"><meta name="apple-mobile-web-app-capable" content="yes"><meta name="apple-mobile-web-app-status-bar-style" content="black-translucent"><meta name="apple-mobile-web-app-title" content="dstackai"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-640x1136.png"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1136x640.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-750x1334.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1334x750.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1125x2436.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2436x1125.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1170x2532.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2532x1170.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1179x2556.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2556x1179.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-828x1792.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1792x828.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2688.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2688x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2208.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2208x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1284x2778.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2778x1284.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1290x2796.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2796x1290.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1488x2266.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2266x1488.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1536x2048.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2048x1536.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1620x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1620.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1640x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1640.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2388.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2388x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2224.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2224x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-2048x2732.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2732x2048.png"><meta name="msapplication-TileColor" content="#fff"><meta name="msapplication-TileImage" content="/assets/mstile-144x144.png"><meta name="msapplication-config" content="/assets/browserconfig.xml"><link rel="yandex-tableau-widget" href="/assets/yandex-browser-manifest.json"><script defer="defer" src="/main-4a0fe83e84574654e397.js"></script><link href="/main-da9f8c06a69c20dac23e.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div class="b-page-header" id="header"></div><div id="root"></div></body></html>