dstack 0.19.33__py3-none-any.whl → 0.19.34__py3-none-any.whl

dstack/_internal/core/backends/base/offers.py

@@ -23,7 +23,8 @@ SUPPORTED_GPUHUNT_FLAGS = [
     "oci-spot",
     "lambda-arm",
     "gcp-a4",
-    "gcp-g4-preview",
+    "gcp-g4",
+    "gcp-dws-calendar-mode",
 ]
 
 
@@ -94,6 +95,7 @@ def catalog_item_to_offer(
         ),
         region=item.location,
         price=item.price,
+        backend_data=item.provider_data,
     )
 
 

dstack/_internal/core/backends/gcp/compute.py

@@ -90,6 +90,10 @@ RESOURCE_NAME_PATTERN = re.compile(r"[a-z0-9-]+")
 TPU_VERSIONS = [tpu.name for tpu in KNOWN_TPUS]
 
 
+class GCPOfferBackendData(CoreModel):
+    is_dws_calendar_mode: bool = False
+
+
 class GCPVolumeDiskBackendData(CoreModel):
     type: Literal["disk"] = "disk"
     disk_type: str
@@ -142,19 +146,13 @@ class GCPCompute(
         offer_keys_to_offers = {}
         offers_with_availability = []
         for offer in offers:
-            preview = False
-            if offer.instance.name.startswith("g4-standard-"):
-                if self.config.preview_features and "g4" in self.config.preview_features:
-                    preview = True
-                else:
-                    continue
             region = offer.region[:-2]  # strip zone
             key = (_unique_instance_name(offer.instance), region)
             if key in offer_keys_to_offers:
                 offer_keys_to_offers[key].availability_zones.append(offer.region)
                 continue
             availability = InstanceAvailability.NO_QUOTA
-            if preview or _has_gpu_quota(quotas[region], offer.instance.resources):
+            if _has_gpu_quota(quotas[region], offer.instance.resources):
                 availability = InstanceAvailability.UNKNOWN
             # todo quotas: cpu, memory, global gpu, tpu
             offer_with_availability = InstanceOfferWithAvailability(
@@ -202,6 +200,23 @@ class GCPCompute(
         modifiers.append(get_offers_disk_modifier(CONFIGURABLE_DISK_SIZE, requirements))
         return modifiers
 
+    def get_offers_post_filter(
+        self, requirements: Requirements
+    ) -> Optional[Callable[[InstanceOfferWithAvailability], bool]]:
+        if requirements.reservation is None:
+
+            def reserved_offers_filter(offer: InstanceOfferWithAvailability) -> bool:
+                """Remove reserved-only offers"""
+                if GCPOfferBackendData.__response__.parse_obj(
+                    offer.backend_data
+                ).is_dws_calendar_mode:
+                    return False
+                return True
+
+            return reserved_offers_filter
+
+        return None
+
     def terminate_instance(
         self, instance_id: str, region: str, backend_data: Optional[str] = None
     ) -> None:
@@ -1006,8 +1021,8 @@ def _has_gpu_quota(quotas: Dict[str, float], resources: Resources) -> bool:
     gpu = resources.gpus[0]
     if _is_tpu(gpu.name):
         return True
-    if gpu.name in ["B200", "H100"]:
-        # B200, H100 and H100_MEGA quotas are not returned by `regions_client.list`
+    if gpu.name in ["B200", "H100", "RTXPRO6000"]:
+        # B200, H100, H100_MEGA, and RTXPRO6000 quotas are not returned by `regions_client.list`
         return True
     quota_name = f"NVIDIA_{gpu.name}_GPUS"
     if gpu.name == "A100" and gpu.memory_mib == 80 * 1024:

dstack/_internal/core/backends/gcp/models.py

@@ -92,7 +92,10 @@ class GCPBackendConfig(CoreModel):
     preview_features: Annotated[
         Optional[List[Literal["g4"]]],
         Field(
-            description=("The list of preview GCP features to enable. Supported values: `g4`"),
+            description=(
+                "The list of preview GCP features to enable."
+                " There are currently no preview features"
+            ),
             max_items=1,
         ),
     ] = None

dstack/_internal/core/backends/runpod/compute.py

@@ -232,9 +232,12 @@ class RunpodCompute(
     def create_volume(self, volume: Volume) -> VolumeProvisioningData:
         volume_name = generate_unique_volume_name(volume, max_length=MAX_RESOURCE_NAME_LEN)
         size_gb = volume.configuration.size_gb
+        # Runpod regions must be uppercase.
+        # Lowercase regions are accepted in the API but they break Runpod in several ways.
+        region = volume.configuration.region.upper()
         volume_id = self.api_client.create_network_volume(
             name=volume_name,
-            region=volume.configuration.region,
+            region=region,
             size=size_gb,
         )
         return VolumeProvisioningData(

dstack/_internal/core/compatibility/runs.py

@@ -33,6 +33,8 @@ def get_apply_plan_excludes(plan: ApplyRunPlanInput) -> Optional[IncludeExcludeD
             current_resource_excludes["deployment_num"] = True
         if current_resource.fleet is None:
             current_resource_excludes["fleet"] = True
+        if current_resource.next_triggered_at is None:
+            current_resource_excludes["next_triggered_at"] = True
         apply_plan_excludes["current_resource"] = current_resource_excludes
         current_resource_excludes["run_spec"] = get_run_spec_excludes(current_resource.run_spec)
         job_submissions_excludes: IncludeExcludeDictType = {}
@@ -47,10 +49,20 @@ def get_apply_plan_excludes(plan: ApplyRunPlanInput) -> Optional[IncludeExcludeD
             job_submissions_excludes["job_provisioning_data"] = {
                 "instance_type": {"resources": {"cpu_arch"}}
             }
+        jrd_offer_excludes = {}
+        if any(
+            js.job_runtime_data and js.job_runtime_data.offer for js in job_submissions
+        ) and all(
+            not js.job_runtime_data
+            or not js.job_runtime_data.offer
+            or not js.job_runtime_data.offer.backend_data
+            for js in job_submissions
+        ):
+            jrd_offer_excludes["backend_data"] = True
         if all(map(_should_exclude_job_submission_jrd_cpu_arch, job_submissions)):
-            job_submissions_excludes["job_runtime_data"] = {
-                "offer": {"instance": {"resources": {"cpu_arch"}}}
-            }
+            jrd_offer_excludes["instance"] = {"resources": {"cpu_arch"}}
+        if jrd_offer_excludes:
+            job_submissions_excludes["job_runtime_data"] = {"offer": jrd_offer_excludes}
         if all(js.exit_status is None for js in job_submissions):
             job_submissions_excludes["exit_status"] = True
         if all(js.status_message == "" for js in job_submissions):
@@ -69,9 +81,18 @@ def get_apply_plan_excludes(plan: ApplyRunPlanInput) -> Optional[IncludeExcludeD
                 latest_job_submission_excludes["job_provisioning_data"] = {
                     "instance_type": {"resources": {"cpu_arch"}}
                 }
+            latest_job_submission_jrd_offer_excludes = {}
+            if (
+                latest_job_submission.job_runtime_data
+                and latest_job_submission.job_runtime_data.offer
+                and not latest_job_submission.job_runtime_data.offer.backend_data
+            ):
+                latest_job_submission_jrd_offer_excludes["backend_data"] = True
             if _should_exclude_job_submission_jrd_cpu_arch(latest_job_submission):
+                latest_job_submission_jrd_offer_excludes["instance"] = {"resources": {"cpu_arch"}}
+            if latest_job_submission_jrd_offer_excludes:
                 latest_job_submission_excludes["job_runtime_data"] = {
-                    "offer": {"instance": {"resources": {"cpu_arch"}}}
+                    "offer": latest_job_submission_jrd_offer_excludes
                 }
             if latest_job_submission.exit_status is None:
                 latest_job_submission_excludes["exit_status"] = True

dstack/_internal/core/models/instances.py

@@ -1,6 +1,6 @@
 import datetime
 from enum import Enum
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
 from uuid import UUID
 
 import gpuhunt
@@ -184,6 +184,7 @@ class InstanceOffer(CoreModel):
     instance: InstanceType
     region: str
     price: float
+    backend_data: dict[str, Any] = {}
 
 
 class InstanceOfferWithAvailability(InstanceOffer):

dstack/_internal/core/models/runs.py

@@ -553,6 +553,7 @@ class Run(CoreModel):
     deployment_num: int = 0  # default for compatibility with pre-0.19.14 servers
     error: Optional[str] = None
     deleted: Optional[bool] = None
+    next_triggered_at: Optional[datetime] = None
 
     def is_deployment_in_progress(self) -> bool:
         return any(

dstack/_internal/core/services/ssh/key_manager.py

@@ -0,0 +1,56 @@
+import os
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import TYPE_CHECKING, Optional
+
+from dstack._internal.core.models.users import UserWithCreds
+
+if TYPE_CHECKING:
+    from dstack.api.server import APIClient
+
+KEY_REFRESH_RATE = timedelta(minutes=10)  # redownload the key periodically in case it was rotated
+
+
+@dataclass
+class UserSSHKey:
+    public_key: str
+    private_key_path: Path
+
+
+class UserSSHKeyManager:
+    def __init__(self, api_client: "APIClient", ssh_keys_dir: Path) -> None:
+        self._api_client = api_client
+        self._key_path = ssh_keys_dir / api_client.get_token_hash()
+        self._pub_key_path = self._key_path.with_suffix(".pub")
+
+    def get_user_key(self) -> Optional[UserSSHKey]:
+        """
+        Return the up-to-date user key, or None if the user has no key (if created before 0.19.33)
+        """
+        if (
+            not self._key_path.exists()
+            or not self._pub_key_path.exists()
+            or datetime.now() - datetime.fromtimestamp(self._key_path.stat().st_mtime)
+            > KEY_REFRESH_RATE
+        ):
+            if not self._download_user_key():
+                return None
+        return UserSSHKey(
+            public_key=self._pub_key_path.read_text(), private_key_path=self._key_path
+        )
+
+    def _download_user_key(self) -> bool:
+        user = self._api_client.users.get_my_user()
+        if not (isinstance(user, UserWithCreds) and user.ssh_public_key and user.ssh_private_key):
+            return False
+
+        def key_opener(path, flags):
+            return os.open(path, flags, 0o600)
+
+        with open(self._key_path, "w", opener=key_opener) as f:
+            f.write(user.ssh_private_key)
+        with open(self._pub_key_path, "w") as f:
+            f.write(user.ssh_public_key)
+
+        return True

dstack/_internal/server/models.py

@@ -190,6 +190,9 @@ class UserModel(BaseModel):
     # deactivated users cannot access API
     active: Mapped[bool] = mapped_column(Boolean, default=True)
 
+    # SSH keys can be null for users created before 0.19.33.
+    # Keys for those users are being gradually generated on /get_my_user calls.
+    # TODO: make keys required in a future version.
     ssh_private_key: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
     ssh_public_key: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
 

dstack/_internal/server/routers/metrics.py

@@ -1,5 +1,6 @@
 from datetime import datetime
 from typing import Optional, Tuple
+from uuid import UUID
 
 from fastapi import APIRouter, Depends
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -29,6 +30,7 @@ router = APIRouter(
 )
 async def get_job_metrics(
     run_name: str,
+    run_id: Optional[UUID] = None,
     replica_num: int = 0,
     job_num: int = 0,
     limit: int = 1,
@@ -39,8 +41,9 @@ async def get_job_metrics(
 ):
     """
     Returns job-level metrics such as hardware utilization
-    given `run_name`, `replica_num`, and `job_num`.
-    If only `run_name` is specified, returns metrics of `(replica_num=0, job_num=0)`.
+    given `run_name`, `run_id`, `replica_num`, and `job_num`.
+    If only `run_name` is specified, returns metrics of `(replica_num=0, job_num=0)`
+    of the latest run with the given name.
     By default, returns one latest sample. To control time window/number of samples, use
     `limit`, `after`, `before`.
 
@@ -61,6 +64,7 @@ async def get_job_metrics(
         session=session,
         project=project,
         run_name=run_name,
+        run_id=run_id,
         replica_num=replica_num,
         job_num=job_num,
     )

dstack/_internal/server/routers/users.py

@@ -38,8 +38,15 @@ async def list_users(
 
 @router.post("/get_my_user", response_model=UserWithCreds)
 async def get_my_user(
+    session: AsyncSession = Depends(get_session),
     user: UserModel = Depends(Authenticated()),
 ):
+    if user.ssh_private_key is None or user.ssh_public_key is None:
+        # Generate keys for pre-0.19.33 users
+        updated_user = await users.refresh_ssh_key(session=session, user=user, username=user.name)
+        if updated_user is None:
+            raise ResourceNotExistsError()
+        user = updated_user
     return CustomORJSONResponse(users.user_model_to_user_with_creds(user))
 
 

dstack/_internal/server/services/jobs/__init__.py

@@ -97,19 +97,28 @@ def find_job(jobs: List[Job], replica_num: int, job_num: int) -> Job:
 
 
 async def get_run_job_model(
-    session: AsyncSession, project: ProjectModel, run_name: str, replica_num: int, job_num: int
+    session: AsyncSession,
+    project: ProjectModel,
+    run_name: str,
+    run_id: Optional[UUID],
+    replica_num: int,
+    job_num: int,
 ) -> Optional[JobModel]:
+    filters = [
+        RunModel.project_id == project.id,
+        RunModel.run_name == run_name,
+        JobModel.replica_num == replica_num,
+        JobModel.job_num == job_num,
+    ]
+    if run_id is not None:
+        filters.append(RunModel.id == run_id)
+    else:
+        # Assuming run_name is unique for non-deleted runs
+        filters.append(RunModel.deleted == False)
     res = await session.execute(
         select(JobModel)
         .join(JobModel.run)
-        .where(
-            RunModel.project_id == project.id,
-            # assuming run_name is unique for non-deleted runs
-            RunModel.run_name == run_name,
-            RunModel.deleted == False,
-            JobModel.replica_num == replica_num,
-            JobModel.job_num == job_num,
-        )
+        .where(*filters)
         .order_by(JobModel.submission_num.desc())
         .limit(1)
     )

dstack/_internal/server/services/offers.py

@@ -215,6 +215,7 @@ def generate_shared_offer(
         ),
         region=offer.region,
         price=offer.price,
+        backend_data=offer.backend_data,
         availability=offer.availability,
         blocks=blocks,
         total_blocks=total_blocks,

dstack/_internal/server/services/runs.py

@@ -715,6 +715,9 @@ def run_model_to_run(
     status_message = _get_run_status_message(run_model)
     error = _get_run_error(run_model)
     fleet = _get_run_fleet(run_model)
+    next_triggered_at = None
+    if not run_model.status.is_finished():
+        next_triggered_at = _get_next_triggered_at(run_spec)
     run = Run(
         id=run_model.id,
         project_name=run_model.project.name,
@@ -734,6 +737,7 @@ def run_model_to_run(
         deployment_num=run_model.deployment_num,
         error=error,
         deleted=run_model.deleted,
+        next_triggered_at=next_triggered_at,
     )
     run.cost = _get_run_cost(run)
     return run

dstack/_internal/server/services/users.py

@@ -20,8 +20,8 @@ from dstack._internal.core.models.users import (
 from dstack._internal.server.models import DecryptedString, UserModel
 from dstack._internal.server.services.permissions import get_default_permissions
 from dstack._internal.server.utils.routers import error_forbidden
+from dstack._internal.utils import crypto
 from dstack._internal.utils.common import run_async
-from dstack._internal.utils.crypto import generate_rsa_key_pair_bytes
 from dstack._internal.utils.logging import get_logger
 
 logger = get_logger(__name__)
@@ -88,7 +88,7 @@ async def create_user(
         raise ResourceExistsError()
     if token is None:
         token = str(uuid.uuid4())
-    private_bytes, public_bytes = await run_async(generate_rsa_key_pair_bytes, username)
+    private_bytes, public_bytes = await run_async(crypto.generate_rsa_key_pair_bytes, username)
     user = UserModel(
         id=uuid.uuid4(),
         name=username,
@@ -135,7 +135,7 @@ async def refresh_ssh_key(
     logger.debug("Refreshing SSH key for user [code]%s[/code]", username)
     if user.global_role != GlobalRole.ADMIN and user.name != username:
         raise error_forbidden()
-    private_bytes, public_bytes = await run_async(generate_rsa_key_pair_bytes, username)
+    private_bytes, public_bytes = await run_async(crypto.generate_rsa_key_pair_bytes, username)
     await session.execute(
         update(UserModel)
         .where(UserModel.name == username)

dstack/_internal/server/statics/index.html

@@ -1,3 +1,3 @@
 <!doctype html><html lang="en"><head><meta charset="utf-8"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><title>dstack</title><meta name="description" content="Get GPUs at the best prices and availability from a wide range of providers. No cloud account of your own is required.
 "/><link rel="preconnect" href="https://fonts.googleapis.com"><link rel="preconnect" href="https://fonts.gstatic.com" crossorigin><link href="https://fonts.googleapis.com/css2?family=Roboto:ital,wght@0,100;0,300;0,400;0,500;0,700;0,900;1,100;1,300;1,400;1,500;1,700;1,900&display=swap" rel="stylesheet"><meta name="og:title" content="dstack"><meta name="og:type" content="article"><meta name="og:image" content="/splash_thumbnail.png"><meta name="og:description" content="Get GPUs at the best prices and availability from a wide range of providers. No cloud account of your own is required.
-"><link rel="icon" type="image/x-icon" href="/assets/favicon.ico"><link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png"><link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png"><link rel="icon" type="image/png" sizes="48x48" href="/assets/favicon-48x48.png"><link rel="manifest" href="/assets/manifest.webmanifest"><meta name="mobile-web-app-capable" content="yes"><meta name="theme-color" content="#fff"><meta name="application-name" content="dstackai"><link rel="apple-touch-icon" sizes="57x57" href="/assets/apple-touch-icon-57x57.png"><link rel="apple-touch-icon" sizes="60x60" href="/assets/apple-touch-icon-60x60.png"><link rel="apple-touch-icon" sizes="72x72" href="/assets/apple-touch-icon-72x72.png"><link rel="apple-touch-icon" sizes="76x76" href="/assets/apple-touch-icon-76x76.png"><link rel="apple-touch-icon" sizes="114x114" href="/assets/apple-touch-icon-114x114.png"><link rel="apple-touch-icon" sizes="120x120" href="/assets/apple-touch-icon-120x120.png"><link rel="apple-touch-icon" sizes="144x144" href="/assets/apple-touch-icon-144x144.png"><link rel="apple-touch-icon" sizes="152x152" href="/assets/apple-touch-icon-152x152.png"><link rel="apple-touch-icon" sizes="167x167" href="/assets/apple-touch-icon-167x167.png"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon-180x180.png"><link rel="apple-touch-icon" sizes="1024x1024" href="/assets/apple-touch-icon-1024x1024.png"><meta name="apple-mobile-web-app-capable" content="yes"><meta name="apple-mobile-web-app-status-bar-style" content="black-translucent"><meta name="apple-mobile-web-app-title" content="dstackai"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-640x1136.png"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1136x640.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-750x1334.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1334x750.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1125x2436.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2436x1125.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1170x2532.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2532x1170.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1179x2556.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2556x1179.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-828x1792.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1792x828.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2688.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2688x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2208.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2208x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1284x2778.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2778x1284.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1290x2796.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2796x1290.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1488x2266.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2266x1488.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1536x2048.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2048x1536.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1620x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1620.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1640x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1640.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2388.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2388x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2224.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2224x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-2048x2732.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2732x2048.png"><meta name="msapplication-TileColor" content="#fff"><meta name="msapplication-TileImage" content="/assets/mstile-144x144.png"><meta name="msapplication-config" content="/assets/browserconfig.xml"><link rel="yandex-tableau-widget" href="/assets/yandex-browser-manifest.json"><script defer="defer" src="/main-97c7e184573ca23f9fe4.js"></script><link href="/main-720ce3a11140daa480cc.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div class="b-page-header" id="header"></div><div id="root"></div><script async src="https://widget.kapa.ai/kapa-widget.bundle.js" data-website-id="11a9339d-20ce-4ddb-9ba3-1b6e29afe8eb" data-project-name="dstack" data-project-color="rgba(0, 0, 0, 0.87)" data-font-size-lg="0.78rem" data-button-hide="true" data-modal-image="/logo-notext.svg" data-modal-z-index="1100" data-modal-title="Ask me anything" data-font-family='metro-web, Metro, -apple-system, "system-ui", "Segoe UI", Roboto' data-project-logo="/assets/images/kapa.svg" data-modal-disclaimer="This is a custom LLM for dstack with access to Documentation, API references and GitHub issues. This feature is experimental - Give it a try!" data-user-analytics-fingerprint-enabled="true"></script></body></html>
+"><link rel="icon" type="image/x-icon" href="/assets/favicon.ico"><link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png"><link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png"><link rel="icon" type="image/png" sizes="48x48" href="/assets/favicon-48x48.png"><link rel="manifest" href="/assets/manifest.webmanifest"><meta name="mobile-web-app-capable" content="yes"><meta name="theme-color" content="#fff"><meta name="application-name" content="dstackai"><link rel="apple-touch-icon" sizes="57x57" href="/assets/apple-touch-icon-57x57.png"><link rel="apple-touch-icon" sizes="60x60" href="/assets/apple-touch-icon-60x60.png"><link rel="apple-touch-icon" sizes="72x72" href="/assets/apple-touch-icon-72x72.png"><link rel="apple-touch-icon" sizes="76x76" href="/assets/apple-touch-icon-76x76.png"><link rel="apple-touch-icon" sizes="114x114" href="/assets/apple-touch-icon-114x114.png"><link rel="apple-touch-icon" sizes="120x120" href="/assets/apple-touch-icon-120x120.png"><link rel="apple-touch-icon" sizes="144x144" href="/assets/apple-touch-icon-144x144.png"><link rel="apple-touch-icon" sizes="152x152" href="/assets/apple-touch-icon-152x152.png"><link rel="apple-touch-icon" sizes="167x167" href="/assets/apple-touch-icon-167x167.png"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon-180x180.png"><link rel="apple-touch-icon" sizes="1024x1024" href="/assets/apple-touch-icon-1024x1024.png"><meta name="apple-mobile-web-app-capable" content="yes"><meta name="apple-mobile-web-app-status-bar-style" content="black-translucent"><meta name="apple-mobile-web-app-title" content="dstackai"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-640x1136.png"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1136x640.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-750x1334.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1334x750.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1125x2436.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2436x1125.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1170x2532.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2532x1170.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1179x2556.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2556x1179.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-828x1792.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1792x828.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2688.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2688x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2208.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2208x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1284x2778.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2778x1284.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1290x2796.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2796x1290.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1488x2266.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2266x1488.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1536x2048.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2048x1536.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1620x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1620.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1640x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1640.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2388.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2388x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2224.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2224x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-2048x2732.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2732x2048.png"><meta name="msapplication-TileColor" content="#fff"><meta name="msapplication-TileImage" content="/assets/mstile-144x144.png"><meta name="msapplication-config" content="/assets/browserconfig.xml"><link rel="yandex-tableau-widget" href="/assets/yandex-browser-manifest.json"><script defer="defer" src="/main-e79754c136f1d8e4e7e6.js"></script><link href="/main-720ce3a11140daa480cc.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div class="b-page-header" id="header"></div><div id="root"></div><script async src="https://widget.kapa.ai/kapa-widget.bundle.js" data-website-id="11a9339d-20ce-4ddb-9ba3-1b6e29afe8eb" data-project-name="dstack" data-project-color="rgba(0, 0, 0, 0.87)" data-font-size-lg="0.78rem" data-button-hide="true" data-modal-image="/logo-notext.svg" data-modal-z-index="1100" data-modal-title="Ask me anything" data-font-family='metro-web, Metro, -apple-system, "system-ui", "Segoe UI", Roboto' data-project-logo="/assets/images/kapa.svg" data-modal-disclaimer="This is a custom LLM for dstack with access to Documentation, API references and GitHub issues. This feature is experimental - Give it a try!" data-user-analytics-fingerprint-enabled="true"></script></body></html>