dstack 0.18.40rc1__py3-none-any.whl → 0.18.42__py3-none-any.whl

dstack/_internal/server/services/runs.py

@@ -29,7 +29,6 @@ from dstack._internal.core.models.runs import (
     ApplyRunPlanInput,
     Job,
     JobPlan,
-    JobProvisioningData,
     JobSpec,
     JobStatus,
     JobSubmission,
@@ -45,8 +44,6 @@ from dstack._internal.core.models.users import GlobalRole
 from dstack._internal.core.models.volumes import (
     InstanceMountPoint,
     Volume,
-    VolumeMountPoint,
-    VolumeStatus,
 )
 from dstack._internal.core.services import validate_dstack_resource_name
 from dstack._internal.core.services.diff import diff_models
@@ -58,15 +55,15 @@ from dstack._internal.server.models import (
     RepoModel,
     RunModel,
     UserModel,
-    VolumeModel,
 )
 from dstack._internal.server.services import repos as repos_services
 from dstack._internal.server.services import services
-from dstack._internal.server.services import volumes as volumes_services
 from dstack._internal.server.services.docker import is_valid_docker_volume_target
 from dstack._internal.server.services.jobs import (
+    check_can_attach_job_volumes,
     delay_job_instance_termination,
     get_instances_ids_with_detaching_volumes,
+    get_job_configured_volumes,
     get_jobs_from_run_spec,
     group_jobs_by_replica_latest,
     job_model_to_job_submission,
@@ -80,6 +77,7 @@ from dstack._internal.server.services.pools import (
     get_instance_offer,
     get_or_create_pool_by_name,
     get_pool_instances,
+    get_shared_pool_instances_with_offers,
 )
 from dstack._internal.server.services.projects import list_project_models, list_user_project_models
 from dstack._internal.server.services.users import get_user_model_by_name
@@ -164,7 +162,8 @@ async def list_projects_run_models(
     limit: int,
     ascending: bool,
 ) -> List[RunModel]:
-    filters = [RunModel.deleted == False, RunModel.project_id.in_(p.id for p in projects)]
+    filters = []
+    filters.append(RunModel.project_id.in_(p.id for p in projects))
     if repo is not None:
         filters.append(RunModel.repo_id == repo.id)
     if runs_user is not None:
@@ -300,10 +299,11 @@ async def get_plan(
     # TODO(egor-s): do we need to generate all replicas here?
     jobs = await get_jobs_from_run_spec(run_spec, replica_num=0)
 
-    volumes = await get_run_volumes(
+    volumes = await get_job_configured_volumes(
         session=session,
         project=project,
         run_spec=run_spec,
+        job_num=0,
     )
 
     pool = await get_or_create_pool_by_name(
@@ -450,7 +450,7 @@ async def submit_run(
         else:
             await delete_runs(session=session, project=project, runs_names=[run_spec.run_name])
 
-        await validate_run(
+        await _validate_run(
             session=session,
             user=user,
             project=project,
@@ -677,27 +677,40 @@ async def _get_pool_offers(
     run_spec: RunSpec,
     job: Job,
     volumes: List[List[Volume]],
-) -> List[InstanceOfferWithAvailability]:
+) -> list[InstanceOfferWithAvailability]:
+    pool_offers: list[InstanceOfferWithAvailability] = []
+
     detaching_instances_ids = await get_instances_ids_with_detaching_volumes(session)
     pool_instances = [i for i in get_pool_instances(pool) if i.id not in detaching_instances_ids]
-    pool_filtered_instances = filter_pool_instances(
+    multinode = job.job_spec.jobs_per_replica > 1
+
+    if not multinode:
+        shared_instances_with_offers = get_shared_pool_instances_with_offers(
+            pool_instances=pool_instances,
+            profile=run_spec.merged_profile,
+            requirements=job.job_spec.requirements,
+            volumes=volumes,
+        )
+        for _, offer in shared_instances_with_offers:
+            pool_offers.append(offer)
+
+    nonshared_instances = filter_pool_instances(
         pool_instances=pool_instances,
         profile=run_spec.merged_profile,
         requirements=job.job_spec.requirements,
-        multinode=job.job_spec.jobs_per_replica > 1,
+        multinode=multinode,
         volumes=volumes,
+        shared=False,
     )
-    pool_offers: List[InstanceOfferWithAvailability] = []
-    for instance in pool_filtered_instances:
+    for instance in nonshared_instances:
         offer = get_instance_offer(instance)
         if offer is None:
             continue
         offer.availability = InstanceAvailability.BUSY
         if instance.status == InstanceStatus.IDLE:
             offer.availability = InstanceAvailability.IDLE
-        if instance.unreachable:
-            offer.availability = InstanceAvailability.NOT_AVAILABLE
         pool_offers.append(offer)
+
     pool_offers.sort(key=lambda offer: offer.price)
     return pool_offers
 
@@ -722,186 +735,42 @@ async def _generate_run_name(
         idx += 1
 
 
-async def validate_run(
-    session: AsyncSession,
-    user: UserModel,
-    project: ProjectModel,
-    run_spec: RunSpec,
-):
-    volumes = await get_run_volumes(
-        session=session,
-        project=project,
-        run_spec=run_spec,
-    )
-    check_can_attach_run_volumes(
-        run_spec=run_spec,
-        volumes=volumes,
+def check_run_spec_requires_instance_mounts(run_spec: RunSpec) -> bool:
+    return any(
+        is_core_model_instance(mp, InstanceMountPoint) and not mp.optional
+        for mp in run_spec.configuration.volumes
     )
 
 
-async def get_run_volumes(
+async def _validate_run(
     session: AsyncSession,
+    user: UserModel,
     project: ProjectModel,
     run_spec: RunSpec,
-) -> List[List[Volume]]:
-    """
-    Returns list of run volumes grouped by mount points.
-    """
-    volume_models = await get_run_volume_models(
+):
+    await _validate_run_volumes(
         session=session,
         project=project,
         run_spec=run_spec,
     )
-    return [
-        [volumes_services.volume_model_to_volume(v) for v in mount_point_volume_models]
-        for mount_point_volume_models in volume_models
-    ]
 
 
-async def get_run_volume_models(
+async def _validate_run_volumes(
     session: AsyncSession,
     project: ProjectModel,
     run_spec: RunSpec,
-) -> List[List[VolumeModel]]:
-    """
-    Returns list of run volume models grouped by mount points.
-    """
-    if len(run_spec.configuration.volumes) == 0:
-        return []
-    volume_models = []
-    for mount_point in run_spec.configuration.volumes:
-        if not is_core_model_instance(mount_point, VolumeMountPoint):
-            continue
-        if isinstance(mount_point.name, str):
-            names = [mount_point.name]
-        else:
-            names = mount_point.name
-        mount_point_volume_models = []
-        for name in names:
-            volume_model = await volumes_services.get_project_volume_model_by_name(
-                session=session,
-                project=project,
-                name=name,
-            )
-            if volume_model is None:
-                raise ResourceNotExistsError(f"Volume {mount_point.name} not found")
-            mount_point_volume_models.append(volume_model)
-        volume_models.append(mount_point_volume_models)
-    return volume_models
-
-
-def check_can_attach_run_volumes(
-    run_spec: RunSpec,
-    volumes: List[List[Volume]],
 ):
-    """
-    Performs basic checks if volumes can be attached.
-    This is useful to show error ASAP (when user submits the run).
-    If the attachment is to fail anyway, the error will be handled when proccessing submitted jobs.
-    """
-    if len(volumes) == 0:
-        return
-    expected_backends = {v.configuration.backend for v in volumes[0]}
-    expected_regions = {v.configuration.region for v in volumes[0]}
-    for mount_point_volumes in volumes:
-        backends = {v.configuration.backend for v in mount_point_volumes}
-        regions = {v.configuration.region for v in mount_point_volumes}
-        if backends != expected_backends:
-            raise ServerClientError(
-                "Volumes from different backends specified for different mount points"
-            )
-        if regions != expected_regions:
-            raise ServerClientError(
-                "Volumes from different regions specified for different mount points"
-            )
-        for volume in mount_point_volumes:
-            if volume.status != VolumeStatus.ACTIVE:
-                raise ServerClientError(f"Cannot mount volumes that are not active: {volume.name}")
-    volumes_names = [v.name for vs in volumes for v in vs]
-    if len(volumes_names) != len(set(volumes_names)):
-        raise ServerClientError("Cannot attach the same volume at different mount points")
-
-
-async def get_job_volumes(
-    session: AsyncSession,
-    project: ProjectModel,
-    run_spec: RunSpec,
-    job_provisioning_data: JobProvisioningData,
-) -> List[Volume]:
-    """
-    Returns volumes attached to the job.
-    """
-    run_volumes = await get_run_volumes(
-        session=session,
-        project=project,
-        run_spec=run_spec,
-    )
-    job_volumes = []
-    for mount_point_volumes in run_volumes:
-        job_volumes.append(get_job_mount_point_volume(mount_point_volumes, job_provisioning_data))
-    return job_volumes
-
-
-def get_job_mount_point_volume(
-    volumes: List[Volume],
-    job_provisioning_data: JobProvisioningData,
-) -> Volume:
-    """
-    Returns the volume attached to the job among the list of possible mount point volumes.
-    """
-    for volume in volumes:
-        if (
-            volume.configuration.backend != job_provisioning_data.get_base_backend()
-            or volume.configuration.region != job_provisioning_data.region
-        ):
-            continue
-        if (
-            volume.provisioning_data is not None
-            and volume.provisioning_data.availability_zone is not None
-            and job_provisioning_data.availability_zone is not None
-            and volume.provisioning_data.availability_zone
-            != job_provisioning_data.availability_zone
-        ):
-            continue
-        return volume
-    raise ServerClientError("Failed to find an eligible volume for the mount point")
-
-
-def get_offer_volumes(
-    volumes: List[List[Volume]],
-    offer: InstanceOfferWithAvailability,
-) -> List[Volume]:
-    """
-    Returns volumes suitable for the offer for each mount point.
-    """
-    offer_volumes = []
-    for mount_point_volumes in volumes:
-        offer_volumes.append(get_offer_mount_point_volume(mount_point_volumes, offer))
-    return offer_volumes
-
-
-def get_offer_mount_point_volume(
-    volumes: List[Volume],
-    offer: InstanceOfferWithAvailability,
-) -> Volume:
-    """
-    Returns the first suitable volume for the offer among possible mount point volumes.
-    """
-    for volume in volumes:
-        if (
-            volume.configuration.backend != offer.backend
-            or volume.configuration.region != offer.region
-        ):
-            continue
-        return volume
-    raise ServerClientError("Failed to find an eligible volume for the mount point")
-
-
-def check_run_spec_requires_instance_mounts(run_spec: RunSpec) -> bool:
-    return any(
-        is_core_model_instance(mp, InstanceMountPoint) and not mp.optional
-        for mp in run_spec.configuration.volumes
-    )
+    # The volumes validation should be done here and not in job configurator
+    # since potentially we may need to validate volumes for jobs/replicas
+    # that won't be created immediately (e.g. range of replicas or nodes).
+    nodes = 1
+    if run_spec.configuration.type == "task":
+        nodes = run_spec.configuration.nodes
+    for job_num in range(nodes):
+        volumes = await get_job_configured_volumes(
+            session=session, project=project, run_spec=run_spec, job_num=job_num
+        )
+        check_can_attach_job_volumes(volumes=volumes)
 
 
 async def _get_run_repo_or_error(

dstack/_internal/server/services/services/__init__.py

@@ -21,6 +21,7 @@ from dstack._internal.core.errors import (
 from dstack._internal.core.models.common import is_core_model_instance
 from dstack._internal.core.models.configurations import SERVICE_HTTPS_DEFAULT, ServiceConfiguration
 from dstack._internal.core.models.gateways import GatewayConfiguration, GatewayStatus
+from dstack._internal.core.models.instances import SSHConnectionParams
 from dstack._internal.core.models.runs import Run, RunSpec, ServiceModelSpec, ServiceSpec
 from dstack._internal.server import settings
 from dstack._internal.server.models import GatewayModel, JobModel, ProjectModel, RunModel
@@ -155,7 +156,12 @@ def get_service_spec(
 
 
 async def register_replica(
-    session: AsyncSession, gateway_id: Optional[uuid.UUID], run: Run, job_model: JobModel
+    session: AsyncSession,
+    gateway_id: Optional[uuid.UUID],
+    run: Run,
+    job_model: JobModel,
+    ssh_head_proxy: Optional[SSHConnectionParams],
+    ssh_head_proxy_private_key: Optional[str],
 ):
     if gateway_id is None:  # in-server proxy
         return
@@ -167,6 +173,8 @@ async def register_replica(
             await client.register_replica(
                 run=run,
                 job_submission=job_submission,
+                ssh_head_proxy=ssh_head_proxy,
+                ssh_head_proxy_private_key=ssh_head_proxy_private_key,
             )
         logger.info("%s: replica is registered for service %s", fmt(job_model), run.id.hex)
     except (httpx.RequestError, SSHError) as e:

dstack/_internal/server/services/volumes.py

@@ -15,19 +15,28 @@ from dstack._internal.core.errors import (
 from dstack._internal.core.models.users import GlobalRole
 from dstack._internal.core.models.volumes import (
     Volume,
+    VolumeAttachment,
     VolumeAttachmentData,
     VolumeConfiguration,
+    VolumeInstance,
     VolumeProvisioningData,
     VolumeStatus,
 )
 from dstack._internal.core.services import validate_dstack_resource_name
 from dstack._internal.server.db import get_db
-from dstack._internal.server.models import ProjectModel, UserModel, VolumeModel
+from dstack._internal.server.models import (
+    InstanceModel,
+    ProjectModel,
+    UserModel,
+    VolumeAttachmentModel,
+    VolumeModel,
+)
 from dstack._internal.server.services import backends as backends_services
 from dstack._internal.server.services.locking import (
     get_locker,
     string_to_lock_id,
 )
+from dstack._internal.server.services.pools import get_instance_provisioning_data
 from dstack._internal.server.services.projects import list_project_models, list_user_project_models
 from dstack._internal.utils import common, random_names
 from dstack._internal.utils.logging import get_logger
@@ -106,8 +115,13 @@ async def list_projects_volume_models(
         .order_by(*order_by)
         .limit(limit)
         .options(joinedload(VolumeModel.user))
+        .options(
+            joinedload(VolumeModel.attachments)
+            .joinedload(VolumeAttachmentModel.instance)
+            .joinedload(InstanceModel.fleet)
+        )
     )
-    volume_models = list(res.scalars().all())
+    volume_models = list(res.unique().scalars().all())
     return volume_models
 
 
@@ -134,9 +148,16 @@ async def list_project_volume_models(
     if not include_deleted:
         filters.append(VolumeModel.deleted == False)
     res = await session.execute(
-        select(VolumeModel).where(*filters).options(joinedload(VolumeModel.user))
+        select(VolumeModel)
+        .where(*filters)
+        .options(joinedload(VolumeModel.user))
+        .options(
+            joinedload(VolumeModel.attachments)
+            .joinedload(VolumeAttachmentModel.instance)
+            .joinedload(InstanceModel.fleet)
+        )
     )
-    return list(res.scalars().all())
+    return list(res.unique().scalars().all())
 
 
 async def get_volume_by_name(
@@ -163,9 +184,16 @@ async def get_project_volume_model_by_name(
     if not include_deleted:
         filters.append(VolumeModel.deleted == False)
     res = await session.execute(
-        select(VolumeModel).where(*filters).options(joinedload(VolumeModel.user))
+        select(VolumeModel)
+        .where(*filters)
+        .options(joinedload(VolumeModel.user))
+        .options(
+            joinedload(VolumeModel.attachments)
+            .joinedload(VolumeAttachmentModel.instance)
+            .joinedload(InstanceModel.fleet)
+        )
     )
-    return res.scalar_one_or_none()
+    return res.unique().scalar_one_or_none()
 
 
 async def create_volume(
@@ -205,10 +233,10 @@ async def create_volume(
             project=project,
             status=VolumeStatus.SUBMITTED,
             configuration=configuration.json(),
+            attachments=[],
         )
         session.add(volume_model)
         await session.commit()
-        await session.refresh(volume_model)
         return volume_model_to_volume(volume_model)
 
 
@@ -234,13 +262,13 @@ async def delete_volumes(session: AsyncSession, project: ProjectModel, names: Li
                 VolumeModel.deleted == False,
             )
             .options(selectinload(VolumeModel.user))
-            .options(selectinload(VolumeModel.instances))
+            .options(selectinload(VolumeModel.attachments))
             .execution_options(populate_existing=True)
             .with_for_update()
         )
         volume_models = res.scalars().unique().all()
         for volume_model in volume_models:
-            if len(volume_model.instances) > 0:
+            if len(volume_model.attachments) > 0:
                 raise ServerClientError(
                     f"Failed to delete volume {volume_model.name}. Volume is in use."
                 )
@@ -270,6 +298,15 @@ def volume_model_to_volume(volume_model: VolumeModel) -> Volume:
     # Initially VolumeProvisionigData lacked backend
     if vpd is not None and vpd.backend is None:
         vpd.backend = configuration.backend
+    attachments = []
+    for volume_attachment_model in volume_model.attachments:
+        instance = volume_attachment_model.instance
+        attachments.append(
+            VolumeAttachment(
+                instance=instance_model_to_volume_instance(instance),
+                attachment_data=get_attachment_data(volume_attachment_model),
+            )
+        )
     return Volume(
         name=volume_model.name,
         project_name=volume_model.project.name,
@@ -282,6 +319,7 @@ def volume_model_to_volume(volume_model: VolumeModel) -> Volume:
         deleted=volume_model.deleted,
         volume_id=vpd.volume_id if vpd is not None else None,
         provisioning_data=vpd,
+        attachments=attachments,
         attachment_data=vad,
         id=volume_model.id,
     )
@@ -303,6 +341,27 @@ def get_volume_attachment_data(volume_model: VolumeModel) -> Optional[VolumeAtta
     return VolumeAttachmentData.__response__.parse_raw(volume_model.volume_attachment_data)
 
 
+def get_attachment_data(
+    volume_attachment_model: VolumeAttachmentModel,
+) -> Optional[VolumeAttachmentData]:
+    if volume_attachment_model.attachment_data is None:
+        return None
+    return VolumeAttachmentData.__response__.parse_raw(volume_attachment_model.attachment_data)
+
+
+def instance_model_to_volume_instance(instance_model: InstanceModel) -> VolumeInstance:
+    instance_id = None
+    jpd = get_instance_provisioning_data(instance_model)
+    if jpd is not None:
+        instance_id = jpd.instance_id
+    return VolumeInstance(
+        name=instance_model.name,
+        fleet_name=instance_model.fleet.name if instance_model.fleet else None,
+        instance_num=instance_model.instance_num,
+        instance_id=instance_id,
+    )
+
+
 async def generate_volume_name(session: AsyncSession, project: ProjectModel) -> str:
     volume_models = await list_project_volume_models(session=session, project=project)
     names = {v.name for v in volume_models}

dstack/_internal/server/statics/index.html

@@ -1,3 +1,3 @@
 <!doctype html><html lang="en"><head><meta charset="utf-8"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><title>dstack</title><meta name="description" content="Get GPUs at the best prices and availability from a wide range of providers. No cloud account of your own is required.
 "/><link rel="preconnect" href="https://fonts.googleapis.com"><link rel="preconnect" href="https://fonts.gstatic.com" crossorigin><link href="https://fonts.googleapis.com/css2?family=Roboto:ital,wght@0,100;0,300;0,400;0,500;0,700;0,900;1,100;1,300;1,400;1,500;1,700;1,900&display=swap" rel="stylesheet"><meta name="og:title" content="dstack"><meta name="og:type" content="article"><meta name="og:image" content="/splash_thumbnail.png"><meta name="og:description" content="Get GPUs at the best prices and availability from a wide range of providers. No cloud account of your own is required.
-"><link rel="icon" type="image/x-icon" href="/assets/favicon.ico"><link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png"><link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png"><link rel="icon" type="image/png" sizes="48x48" href="/assets/favicon-48x48.png"><link rel="manifest" href="/assets/manifest.webmanifest"><meta name="mobile-web-app-capable" content="yes"><meta name="theme-color" content="#fff"><meta name="application-name" content="dstackai"><link rel="apple-touch-icon" sizes="57x57" href="/assets/apple-touch-icon-57x57.png"><link rel="apple-touch-icon" sizes="60x60" href="/assets/apple-touch-icon-60x60.png"><link rel="apple-touch-icon" sizes="72x72" href="/assets/apple-touch-icon-72x72.png"><link rel="apple-touch-icon" sizes="76x76" href="/assets/apple-touch-icon-76x76.png"><link rel="apple-touch-icon" sizes="114x114" href="/assets/apple-touch-icon-114x114.png"><link rel="apple-touch-icon" sizes="120x120" href="/assets/apple-touch-icon-120x120.png"><link rel="apple-touch-icon" sizes="144x144" href="/assets/apple-touch-icon-144x144.png"><link rel="apple-touch-icon" sizes="152x152" href="/assets/apple-touch-icon-152x152.png"><link rel="apple-touch-icon" sizes="167x167" href="/assets/apple-touch-icon-167x167.png"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon-180x180.png"><link rel="apple-touch-icon" sizes="1024x1024" href="/assets/apple-touch-icon-1024x1024.png"><meta name="apple-mobile-web-app-capable" content="yes"><meta name="apple-mobile-web-app-status-bar-style" content="black-translucent"><meta name="apple-mobile-web-app-title" content="dstackai"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-640x1136.png"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1136x640.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-750x1334.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1334x750.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1125x2436.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2436x1125.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1170x2532.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2532x1170.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1179x2556.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2556x1179.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-828x1792.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1792x828.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2688.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2688x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2208.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2208x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1284x2778.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2778x1284.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1290x2796.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2796x1290.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1488x2266.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2266x1488.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1536x2048.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2048x1536.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1620x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1620.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1640x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1640.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2388.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2388x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2224.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2224x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-2048x2732.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2732x2048.png"><meta name="msapplication-TileColor" content="#fff"><meta name="msapplication-TileImage" content="/assets/mstile-144x144.png"><meta name="msapplication-config" content="/assets/browserconfig.xml"><link rel="yandex-tableau-widget" href="/assets/yandex-browser-manifest.json"><script defer="defer" src="/main-11ec5e4a00ea6ec833e3.js"></script><link href="/main-fc56d1f4af8e57522a1c.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div class="b-page-header" id="header"></div><div id="root"></div></body></html>
+"><link rel="icon" type="image/x-icon" href="/assets/favicon.ico"><link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png"><link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png"><link rel="icon" type="image/png" sizes="48x48" href="/assets/favicon-48x48.png"><link rel="manifest" href="/assets/manifest.webmanifest"><meta name="mobile-web-app-capable" content="yes"><meta name="theme-color" content="#fff"><meta name="application-name" content="dstackai"><link rel="apple-touch-icon" sizes="57x57" href="/assets/apple-touch-icon-57x57.png"><link rel="apple-touch-icon" sizes="60x60" href="/assets/apple-touch-icon-60x60.png"><link rel="apple-touch-icon" sizes="72x72" href="/assets/apple-touch-icon-72x72.png"><link rel="apple-touch-icon" sizes="76x76" href="/assets/apple-touch-icon-76x76.png"><link rel="apple-touch-icon" sizes="114x114" href="/assets/apple-touch-icon-114x114.png"><link rel="apple-touch-icon" sizes="120x120" href="/assets/apple-touch-icon-120x120.png"><link rel="apple-touch-icon" sizes="144x144" href="/assets/apple-touch-icon-144x144.png"><link rel="apple-touch-icon" sizes="152x152" href="/assets/apple-touch-icon-152x152.png"><link rel="apple-touch-icon" sizes="167x167" href="/assets/apple-touch-icon-167x167.png"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon-180x180.png"><link rel="apple-touch-icon" sizes="1024x1024" href="/assets/apple-touch-icon-1024x1024.png"><meta name="apple-mobile-web-app-capable" content="yes"><meta name="apple-mobile-web-app-status-bar-style" content="black-translucent"><meta name="apple-mobile-web-app-title" content="dstackai"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-640x1136.png"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1136x640.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-750x1334.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1334x750.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1125x2436.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2436x1125.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1170x2532.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2532x1170.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1179x2556.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2556x1179.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-828x1792.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1792x828.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2688.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2688x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2208.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2208x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1284x2778.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2778x1284.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1290x2796.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2796x1290.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1488x2266.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2266x1488.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1536x2048.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2048x1536.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1620x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1620.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1640x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1640.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2388.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2388x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2224.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2224x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-2048x2732.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2732x2048.png"><meta name="msapplication-TileColor" content="#fff"><meta name="msapplication-TileImage" content="/assets/mstile-144x144.png"><meta name="msapplication-config" content="/assets/browserconfig.xml"><link rel="yandex-tableau-widget" href="/assets/yandex-browser-manifest.json"><script defer="defer" src="/main-2ac66bfcbd2e39830b88.js"></script><link href="/main-ad5150a441de98cd8987.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div class="b-page-header" id="header"></div><div id="root"></div></body></html>