dstack 0.19.0rc1__py3-none-any.whl → 0.19.2__py3-none-any.whl

dstack/_internal/server/services/prometheus.py

@@ -1,40 +1,38 @@
 import itertools
+from collections import defaultdict
 from collections.abc import Generator, Iterable
 from datetime import timezone
+from typing import ClassVar
+from uuid import UUID
 
 from prometheus_client import Metric
 from prometheus_client.parser import text_string_to_metric_families
 from prometheus_client.samples import Sample
-from sqlalchemy import select
+from sqlalchemy import func, select
 from sqlalchemy.ext.asyncio import AsyncSession
-from sqlalchemy.orm import joinedload
+from sqlalchemy.orm import aliased, joinedload
 
 from dstack._internal.core.models.instances import InstanceStatus
-from dstack._internal.core.models.runs import JobStatus, RunSpec
+from dstack._internal.core.models.runs import JobStatus, RunSpec, RunStatus
 from dstack._internal.server.models import (
     InstanceModel,
+    JobMetricsPoint,
     JobModel,
     JobPrometheusMetrics,
     ProjectModel,
     RunModel,
+    UserModel,
 )
 from dstack._internal.server.services.instances import get_instance_offer
 from dstack._internal.server.services.jobs import get_job_provisioning_data, get_job_runtime_data
 from dstack._internal.utils.common import get_current_datetime
 
-_INSTANCE_DURATION = "dstack_instance_duration_seconds_total"
-_INSTANCE_PRICE = "dstack_instance_price_dollars_per_hour"
-_INSTANCE_GPU_COUNT = "dstack_instance_gpu_count"
-_JOB_DURATION = "dstack_job_duration_seconds_total"
-_JOB_PRICE = "dstack_job_price_dollars_per_hour"
-_JOB_GPU_COUNT = "dstack_job_gpu_count"
-
 
 async def get_metrics(session: AsyncSession) -> str:
     metrics_iter = itertools.chain(
         await get_instance_metrics(session),
+        await get_run_metrics(session),
         await get_job_metrics(session),
-        await get_job_gpu_metrics(session),
     )
     return "\n".join(_render_metrics(metrics_iter)) + "\n"
 
@@ -61,19 +59,7 @@ async def get_instance_metrics(session: AsyncSession) -> Iterable[Metric]:
         )
     )
     instances = res.unique().scalars().all()
-    metrics: dict[str, Metric] = {
-        _INSTANCE_DURATION: Metric(
-            name=_INSTANCE_DURATION,
-            documentation="Total seconds the instance is running",
-            typ="counter",
-        ),
-        _INSTANCE_PRICE: Metric(
-            name=_INSTANCE_PRICE, documentation="Instance price, USD/hour", typ="gauge"
-        ),
-        _INSTANCE_GPU_COUNT: Metric(
-            name=_INSTANCE_GPU_COUNT, documentation="Instance GPU count", typ="gauge"
-        ),
-    }
+    metrics = _InstanceMetrics()
     now = get_current_datetime()
     for instance in instances:
         fleet = instance.fleet
@@ -94,15 +80,36 @@ async def get_instance_metrics(session: AsyncSession) -> Iterable[Metric]:
             "dstack_gpu": gpu,
         }
         duration = (now - instance.created_at.replace(tzinfo=timezone.utc)).total_seconds()
-        metrics[_INSTANCE_DURATION].add_sample(
-            name=_INSTANCE_DURATION, labels=labels, value=duration
-        )
-        metrics[_INSTANCE_PRICE].add_sample(
-            name=_INSTANCE_PRICE, labels=labels, value=instance.price or 0.0
-        )
-        metrics[_INSTANCE_GPU_COUNT].add_sample(
-            name=_INSTANCE_GPU_COUNT, labels=labels, value=gpu_count
-        )
+        metrics.add_sample(_INSTANCE_DURATION, labels, duration)
+        metrics.add_sample(_INSTANCE_PRICE, labels, instance.price or 0.0)
+        metrics.add_sample(_INSTANCE_GPU_COUNT, labels, gpu_count)
+    return metrics.values()
+
+
+async def get_run_metrics(session: AsyncSession) -> Iterable[Metric]:
+    res = await session.execute(
+        select(ProjectModel.name, UserModel.name, RunModel.status, func.count(RunModel.id))
+        .join_from(RunModel, ProjectModel)
+        .join_from(RunModel, UserModel, RunModel.user_id == UserModel.id)
+        .group_by(ProjectModel.name, UserModel.name, RunModel.status)
+        .order_by(ProjectModel.name, UserModel.name, RunModel.status)
+    )
+    projects: dict[str, dict[str, dict[RunStatus, int]]] = defaultdict(
+        lambda: defaultdict(lambda: defaultdict(int))
+    )
+    for project_name, user_name, status, count in res.all():
+        projects[project_name][user_name][status] = count
+    metrics = _RunMetrics()
+    for project_name, users in projects.items():
+        for user_name, statuses in users.items():
+            labels: dict[str, str] = {
+                "dstack_project_name": project_name,
+                "dstack_user_name": user_name,
+            }
+            metrics.add_sample(_RUN_COUNT_TOTAL, labels, sum(statuses.values()))
+            metrics.add_sample(_RUN_COUNT_TERMINATED, labels, statuses[RunStatus.TERMINATED])
+            metrics.add_sample(_RUN_COUNT_FAILED, labels, statuses[RunStatus.FAILED])
+            metrics.add_sample(_RUN_COUNT_DONE, labels, statuses[RunStatus.DONE])
     return metrics.values()
 
 
@@ -127,106 +134,162 @@ async def get_job_metrics(session: AsyncSession) -> Iterable[Metric]:
         )
     )
     jobs = res.scalars().all()
-    metrics: dict[str, Metric] = {
-        _JOB_DURATION: Metric(
-            name=_JOB_DURATION, documentation="Total seconds the job is running", typ="counter"
-        ),
-        _JOB_PRICE: Metric(
-            name=_JOB_PRICE, documentation="Job instance price, USD/hour", typ="gauge"
-        ),
-        _JOB_GPU_COUNT: Metric(name=_JOB_GPU_COUNT, documentation="Job GPU count", typ="gauge"),
-    }
+    job_ids = {job.id for job in jobs}
+    job_metrics_points = await _get_job_metrics_points(session, job_ids)
+    job_prometheus_metrics = await _get_job_prometheus_metrics(session, job_ids)
+
+    metrics = _JobMetrics()
     now = get_current_datetime()
     for job in jobs:
         jpd = get_job_provisioning_data(job)
         if jpd is None:
             continue
         jrd = get_job_runtime_data(job)
-        gpus = jpd.instance_type.resources.gpus
+        resources = jpd.instance_type.resources
         price = jpd.price
         if jrd is not None and jrd.offer is not None:
-            gpus = jrd.offer.instance.resources.gpus
+            resources = jrd.offer.instance.resources
             price = jrd.offer.price
+        gpus = resources.gpus
+        cpus = resources.cpus
         run_spec = RunSpec.__response__.parse_raw(job.run.run_spec)
-        labels = _get_job_labels(job)
-        labels["dstack_run_type"] = run_spec.configuration.type
-        labels["dstack_backend"] = jpd.get_base_backend().value
-        labels["dstack_gpu"] = gpus[0].name if gpus else ""
+        labels = {
+            "dstack_project_name": job.project.name,
+            "dstack_user_name": job.run.user.name,
+            "dstack_run_name": job.run_name,
+            "dstack_run_id": str(job.run_id),
+            "dstack_job_name": job.job_name,
+            "dstack_job_id": str(job.id),
+            "dstack_job_num": str(job.job_num),
+            "dstack_replica_num": str(job.replica_num),
+            "dstack_run_type": run_spec.configuration.type,
+            "dstack_backend": jpd.get_base_backend().value,
+            "dstack_gpu": gpus[0].name if gpus else "",
+        }
         duration = (now - job.submitted_at.replace(tzinfo=timezone.utc)).total_seconds()
-        metrics[_JOB_DURATION].add_sample(name=_JOB_DURATION, labels=labels, value=duration)
-        metrics[_JOB_PRICE].add_sample(name=_JOB_PRICE, labels=labels, value=price)
-        metrics[_JOB_GPU_COUNT].add_sample(name=_JOB_GPU_COUNT, labels=labels, value=len(gpus))
+        metrics.add_sample(_JOB_DURATION, labels, duration)
+        metrics.add_sample(_JOB_PRICE, labels, price)
+        metrics.add_sample(_JOB_GPU_COUNT, labels, len(gpus))
+        metrics.add_sample(_JOB_CPU_COUNT, labels, cpus)
+        metrics.add_sample(_JOB_MEMORY_TOTAL, labels, resources.memory_mib * 1024 * 1024)
+        jmp = job_metrics_points.get(job.id)
+        if jmp is not None:
+            metrics.add_sample(_JOB_CPU_TIME, labels, jmp.cpu_usage_micro / 1_000_000)
+            metrics.add_sample(_JOB_MEMORY_USAGE, labels, jmp.memory_usage_bytes)
+            metrics.add_sample(_JOB_MEMORY_WORKING_SET, labels, jmp.memory_working_set_bytes)
+        jpm = job_prometheus_metrics.get(job.id)
+        if jpm is not None:
+            for metric in text_string_to_metric_families(jpm.text):
+                metrics.add_metric(metric, labels)
     return metrics.values()
 
 
-async def get_job_gpu_metrics(session: AsyncSession) -> Iterable[Metric]:
-    res = await session.execute(
-        select(JobPrometheusMetrics)
-        .join(JobModel)
-        .join(ProjectModel)
-        .where(JobModel.status.in_([JobStatus.RUNNING]))
-        .order_by(ProjectModel.name, JobModel.job_name)
-        .options(
-            joinedload(JobPrometheusMetrics.job).joinedload(JobModel.project),
-            joinedload(JobPrometheusMetrics.job)
-            .joinedload(JobModel.run)
-            .joinedload(RunModel.user),
-        )
-    )
-    metrics_models = res.scalars().all()
-    return _parse_and_enrich_job_gpu_metrics(metrics_models)
+_COUNTER = "counter"
+_GAUGE = "gauge"
 
+_INSTANCE_DURATION = "dstack_instance_duration_seconds_total"
+_INSTANCE_PRICE = "dstack_instance_price_dollars_per_hour"
+_INSTANCE_GPU_COUNT = "dstack_instance_gpu_count"
+_RUN_COUNT_TOTAL = "dstack_run_count_total"
+_RUN_COUNT_TERMINATED = "dstack_run_count_terminated_total"
+_RUN_COUNT_FAILED = "dstack_run_count_failed_total"
+_RUN_COUNT_DONE = "dstack_run_count_done_total"
+_JOB_DURATION = "dstack_job_duration_seconds_total"
+_JOB_PRICE = "dstack_job_price_dollars_per_hour"
+_JOB_GPU_COUNT = "dstack_job_gpu_count"
+_JOB_CPU_COUNT = "dstack_job_cpu_count"
+_JOB_CPU_TIME = "dstack_job_cpu_time_seconds_total"
+_JOB_MEMORY_TOTAL = "dstack_job_memory_total_bytes"
+_JOB_MEMORY_USAGE = "dstack_job_memory_usage_bytes"
+_JOB_MEMORY_WORKING_SET = "dstack_job_memory_working_set_bytes"
 
-async def get_project_metrics(session: AsyncSession, project: ProjectModel) -> str:
-    res = await session.execute(
-        select(JobPrometheusMetrics)
-        .join(JobModel)
-        .where(
-            JobModel.project_id == project.id,
-            JobModel.status.in_([JobStatus.RUNNING]),
+
+class _Metrics(dict[str, Metric]):
+    metrics: ClassVar[list[tuple[str, str, str]]]
+
+    def __init__(self):
+        super().__init__()
+        for name, typ, documentation in self.metrics:
+            self[name] = Metric(name=name, documentation=documentation, typ=typ)
+
+    def add_sample(self, name: str, labels: dict[str, str], value: float) -> None:
+        # NOTE: Keeps reference to labels.
+        self[name].add_sample(name=name, labels=labels, value=value)
+
+    def add_metric(self, metric: Metric, labels: dict[str, str]) -> None:
+        # NOTE: Modifies and keeps reference to metric.
+        name = metric.name
+        samples = metric.samples
+        stored_metric = self.get(name)
+        if stored_metric is None:
+            stored_metric = metric
+            stored_metric.samples = []
+            self[name] = stored_metric
+        for sample in samples:
+            sample.labels.update(labels)
+            # text_string_to_metric_families "fixes" counter names appending _total,
+            # we rebuild Sample to revert this
+            stored_metric.samples.append(Sample(name, *sample[1:]))
+
+
+class _InstanceMetrics(_Metrics):
+    metrics = [
+        (_INSTANCE_DURATION, _COUNTER, "Total seconds the instance is running"),
+        (_INSTANCE_PRICE, _GAUGE, "Instance price, USD/hour"),
+        (_INSTANCE_GPU_COUNT, _GAUGE, "Instance GPU count"),
+    ]
+
+
+class _RunMetrics(_Metrics):
+    metrics = [
+        (_RUN_COUNT_TOTAL, _COUNTER, "Total runs count"),
+        (_RUN_COUNT_TERMINATED, _COUNTER, "Terminated runs count"),
+        (_RUN_COUNT_FAILED, _COUNTER, "Failed runs count"),
+        (_RUN_COUNT_DONE, _COUNTER, "Done runs count"),
+    ]
+
+
+class _JobMetrics(_Metrics):
+    metrics = [
+        (_JOB_DURATION, _COUNTER, "Total seconds the job is running"),
+        (_JOB_PRICE, _GAUGE, "Job instance price, USD/hour"),
+        (_JOB_GPU_COUNT, _GAUGE, "Job GPU count"),
+        (_JOB_CPU_COUNT, _GAUGE, "Job CPU count"),
+        (_JOB_CPU_TIME, _COUNTER, "Total CPU time consumed by the job, seconds"),
+        (_JOB_MEMORY_TOTAL, _GAUGE, "Total memory allocated for the job, bytes"),
+        (_JOB_MEMORY_USAGE, _GAUGE, "Memory used by the job (including cache), bytes"),
+        (_JOB_MEMORY_WORKING_SET, _GAUGE, "Memory used by the job (not including cache), bytes"),
+    ]
+
+
+async def _get_job_metrics_points(
+    session: AsyncSession, job_ids: Iterable[UUID]
+) -> dict[UUID, JobMetricsPoint]:
+    subquery = select(
+        JobMetricsPoint,
+        func.row_number()
+        .over(
+            partition_by=JobMetricsPoint.job_id,
+            order_by=JobMetricsPoint.timestamp_micro.desc(),
         )
-        .order_by(JobModel.job_name)
-        .options(
-            joinedload(JobPrometheusMetrics.job).joinedload(JobModel.project),
-            joinedload(JobPrometheusMetrics.job)
-            .joinedload(JobModel.run)
-            .joinedload(RunModel.user),
+        .label("row_number"),
+    ).subquery()
+    res = await session.execute(
+        select(aliased(JobMetricsPoint, subquery)).where(
+            subquery.c.row_number == 1,
+            subquery.c.job_id.in_(job_ids),
         )
     )
-    metrics_models = res.scalars().all()
-    return "\n".join(_render_metrics(_parse_and_enrich_job_gpu_metrics(metrics_models))) + "\n"
-
-
-def _parse_and_enrich_job_gpu_metrics(
-    metrics_models: Iterable[JobPrometheusMetrics],
-) -> Iterable[Metric]:
-    metrics: dict[str, Metric] = {}
-    for metrics_model in metrics_models:
-        for metric in text_string_to_metric_families(metrics_model.text):
-            samples = metric.samples
-            metric.samples = []
-            name = metric.name
-            metric = metrics.setdefault(name, metric)
-            for sample in samples:
-                labels = sample.labels
-                labels.update(_get_job_labels(metrics_model.job))
-                # text_string_to_metric_families "fixes" counter names appending _total,
-                # we rebuild Sample to revert this
-                metric.samples.append(Sample(name, labels, *sample[2:]))
-    return metrics.values()
+    return {p.job_id: p for p in res.scalars().all()}
 
 
-def _get_job_labels(job: JobModel) -> dict[str, str]:
-    return {
-        "dstack_project_name": job.project.name,
-        "dstack_user_name": job.run.user.name,
-        "dstack_run_name": job.run_name,
-        "dstack_run_id": str(job.run_id),
-        "dstack_job_name": job.job_name,
-        "dstack_job_id": str(job.id),
-        "dstack_job_num": str(job.job_num),
-        "dstack_replica_num": str(job.replica_num),
-    }
+async def _get_job_prometheus_metrics(
+    session: AsyncSession, job_ids: Iterable[UUID]
+) -> dict[UUID, JobPrometheusMetrics]:
+    res = await session.execute(
+        select(JobPrometheusMetrics).where(JobPrometheusMetrics.job_id.in_(job_ids))
+    )
+    return {p.job_id: p for p in res.scalars().all()}
 
 
 def _render_metrics(metrics: Iterable[Metric]) -> Generator[str, None, None]:

dstack/_internal/server/statics/index.html

@@ -1,3 +1,3 @@
 <!doctype html><html lang="en"><head><meta charset="utf-8"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><title>dstack</title><meta name="description" content="Get GPUs at the best prices and availability from a wide range of providers. No cloud account of your own is required.
 "/><link rel="preconnect" href="https://fonts.googleapis.com"><link rel="preconnect" href="https://fonts.gstatic.com" crossorigin><link href="https://fonts.googleapis.com/css2?family=Roboto:ital,wght@0,100;0,300;0,400;0,500;0,700;0,900;1,100;1,300;1,400;1,500;1,700;1,900&display=swap" rel="stylesheet"><meta name="og:title" content="dstack"><meta name="og:type" content="article"><meta name="og:image" content="/splash_thumbnail.png"><meta name="og:description" content="Get GPUs at the best prices and availability from a wide range of providers. No cloud account of your own is required.
-"><link rel="icon" type="image/x-icon" href="/assets/favicon.ico"><link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png"><link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png"><link rel="icon" type="image/png" sizes="48x48" href="/assets/favicon-48x48.png"><link rel="manifest" href="/assets/manifest.webmanifest"><meta name="mobile-web-app-capable" content="yes"><meta name="theme-color" content="#fff"><meta name="application-name" content="dstackai"><link rel="apple-touch-icon" sizes="57x57" href="/assets/apple-touch-icon-57x57.png"><link rel="apple-touch-icon" sizes="60x60" href="/assets/apple-touch-icon-60x60.png"><link rel="apple-touch-icon" sizes="72x72" href="/assets/apple-touch-icon-72x72.png"><link rel="apple-touch-icon" sizes="76x76" href="/assets/apple-touch-icon-76x76.png"><link rel="apple-touch-icon" sizes="114x114" href="/assets/apple-touch-icon-114x114.png"><link rel="apple-touch-icon" sizes="120x120" href="/assets/apple-touch-icon-120x120.png"><link rel="apple-touch-icon" sizes="144x144" href="/assets/apple-touch-icon-144x144.png"><link rel="apple-touch-icon" sizes="152x152" href="/assets/apple-touch-icon-152x152.png"><link rel="apple-touch-icon" sizes="167x167" href="/assets/apple-touch-icon-167x167.png"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon-180x180.png"><link rel="apple-touch-icon" sizes="1024x1024" href="/assets/apple-touch-icon-1024x1024.png"><meta name="apple-mobile-web-app-capable" content="yes"><meta name="apple-mobile-web-app-status-bar-style" content="black-translucent"><meta name="apple-mobile-web-app-title" content="dstackai"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-640x1136.png"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1136x640.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-750x1334.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1334x750.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1125x2436.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2436x1125.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1170x2532.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2532x1170.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1179x2556.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2556x1179.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-828x1792.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1792x828.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2688.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2688x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2208.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2208x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1284x2778.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2778x1284.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1290x2796.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2796x1290.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1488x2266.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2266x1488.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1536x2048.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2048x1536.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1620x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1620.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1640x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1640.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2388.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2388x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2224.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2224x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-2048x2732.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2732x2048.png"><meta name="msapplication-TileColor" content="#fff"><meta name="msapplication-TileImage" content="/assets/mstile-144x144.png"><meta name="msapplication-config" content="/assets/browserconfig.xml"><link rel="yandex-tableau-widget" href="/assets/yandex-browser-manifest.json"><script defer="defer" src="/main-4fd5a4770eff59325ee3.js"></script><link href="/main-da9f8c06a69c20dac23e.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div class="b-page-header" id="header"></div><div id="root"></div></body></html>
+"><link rel="icon" type="image/x-icon" href="/assets/favicon.ico"><link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png"><link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png"><link rel="icon" type="image/png" sizes="48x48" href="/assets/favicon-48x48.png"><link rel="manifest" href="/assets/manifest.webmanifest"><meta name="mobile-web-app-capable" content="yes"><meta name="theme-color" content="#fff"><meta name="application-name" content="dstackai"><link rel="apple-touch-icon" sizes="57x57" href="/assets/apple-touch-icon-57x57.png"><link rel="apple-touch-icon" sizes="60x60" href="/assets/apple-touch-icon-60x60.png"><link rel="apple-touch-icon" sizes="72x72" href="/assets/apple-touch-icon-72x72.png"><link rel="apple-touch-icon" sizes="76x76" href="/assets/apple-touch-icon-76x76.png"><link rel="apple-touch-icon" sizes="114x114" href="/assets/apple-touch-icon-114x114.png"><link rel="apple-touch-icon" sizes="120x120" href="/assets/apple-touch-icon-120x120.png"><link rel="apple-touch-icon" sizes="144x144" href="/assets/apple-touch-icon-144x144.png"><link rel="apple-touch-icon" sizes="152x152" href="/assets/apple-touch-icon-152x152.png"><link rel="apple-touch-icon" sizes="167x167" href="/assets/apple-touch-icon-167x167.png"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon-180x180.png"><link rel="apple-touch-icon" sizes="1024x1024" href="/assets/apple-touch-icon-1024x1024.png"><meta name="apple-mobile-web-app-capable" content="yes"><meta name="apple-mobile-web-app-status-bar-style" content="black-translucent"><meta name="apple-mobile-web-app-title" content="dstackai"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-640x1136.png"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1136x640.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-750x1334.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1334x750.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1125x2436.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2436x1125.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1170x2532.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2532x1170.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1179x2556.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2556x1179.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-828x1792.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1792x828.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2688.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2688x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2208.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2208x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1284x2778.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2778x1284.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1290x2796.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2796x1290.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1488x2266.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2266x1488.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1536x2048.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2048x1536.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1620x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1620.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1640x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1640.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2388.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2388x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2224.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2224x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-2048x2732.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2732x2048.png"><meta name="msapplication-TileColor" content="#fff"><meta name="msapplication-TileImage" content="/assets/mstile-144x144.png"><meta name="msapplication-config" content="/assets/browserconfig.xml"><link rel="yandex-tableau-widget" href="/assets/yandex-browser-manifest.json"><script defer="defer" src="/main-bcb3228138bc8483cc0b.js"></script><link href="/main-c0bdaac8f1ea67d499eb.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div class="b-page-header" id="header"></div><div id="root"></div></body></html>