dstack 0.19.32__py3-none-any.whl → 0.19.34__py3-none-any.whl

dstack/_internal/core/compatibility/runs.py

@@ -33,6 +33,8 @@ def get_apply_plan_excludes(plan: ApplyRunPlanInput) -> Optional[IncludeExcludeD
             current_resource_excludes["deployment_num"] = True
         if current_resource.fleet is None:
             current_resource_excludes["fleet"] = True
+        if current_resource.next_triggered_at is None:
+            current_resource_excludes["next_triggered_at"] = True
         apply_plan_excludes["current_resource"] = current_resource_excludes
         current_resource_excludes["run_spec"] = get_run_spec_excludes(current_resource.run_spec)
         job_submissions_excludes: IncludeExcludeDictType = {}
@@ -47,10 +49,20 @@ def get_apply_plan_excludes(plan: ApplyRunPlanInput) -> Optional[IncludeExcludeD
             job_submissions_excludes["job_provisioning_data"] = {
                 "instance_type": {"resources": {"cpu_arch"}}
             }
+        jrd_offer_excludes = {}
+        if any(
+            js.job_runtime_data and js.job_runtime_data.offer for js in job_submissions
+        ) and all(
+            not js.job_runtime_data
+            or not js.job_runtime_data.offer
+            or not js.job_runtime_data.offer.backend_data
+            for js in job_submissions
+        ):
+            jrd_offer_excludes["backend_data"] = True
         if all(map(_should_exclude_job_submission_jrd_cpu_arch, job_submissions)):
-            job_submissions_excludes["job_runtime_data"] = {
-                "offer": {"instance": {"resources": {"cpu_arch"}}}
-            }
+            jrd_offer_excludes["instance"] = {"resources": {"cpu_arch"}}
+        if jrd_offer_excludes:
+            job_submissions_excludes["job_runtime_data"] = {"offer": jrd_offer_excludes}
         if all(js.exit_status is None for js in job_submissions):
             job_submissions_excludes["exit_status"] = True
         if all(js.status_message == "" for js in job_submissions):
@@ -69,9 +81,18 @@ def get_apply_plan_excludes(plan: ApplyRunPlanInput) -> Optional[IncludeExcludeD
                 latest_job_submission_excludes["job_provisioning_data"] = {
                     "instance_type": {"resources": {"cpu_arch"}}
                 }
+            latest_job_submission_jrd_offer_excludes = {}
+            if (
+                latest_job_submission.job_runtime_data
+                and latest_job_submission.job_runtime_data.offer
+                and not latest_job_submission.job_runtime_data.offer.backend_data
+            ):
+                latest_job_submission_jrd_offer_excludes["backend_data"] = True
             if _should_exclude_job_submission_jrd_cpu_arch(latest_job_submission):
+                latest_job_submission_jrd_offer_excludes["instance"] = {"resources": {"cpu_arch"}}
+            if latest_job_submission_jrd_offer_excludes:
                 latest_job_submission_excludes["job_runtime_data"] = {
-                    "offer": {"instance": {"resources": {"cpu_arch"}}}
+                    "offer": latest_job_submission_jrd_offer_excludes
                 }
             if latest_job_submission.exit_status is None:
                 latest_job_submission_excludes["exit_status"] = True

dstack/_internal/core/models/fleets.py

@@ -244,7 +244,7 @@ class InstanceGroupParams(CoreModel):
         Field(
             description=(
                 "The existing reservation to use for instance provisioning."
-                " Supports AWS Capacity Reservations and Capacity Blocks"
+                " Supports AWS Capacity Reservations, AWS Capacity Blocks, and GCP reservations"
             )
         ),
     ] = None

dstack/_internal/core/models/instances.py

@@ -1,6 +1,6 @@
 import datetime
 from enum import Enum
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
 from uuid import UUID
 
 import gpuhunt
@@ -184,6 +184,7 @@ class InstanceOffer(CoreModel):
     instance: InstanceType
     region: str
     price: float
+    backend_data: dict[str, Any] = {}
 
 
 class InstanceOfferWithAvailability(InstanceOffer):

dstack/_internal/core/models/profiles.py

@@ -283,7 +283,7 @@ class ProfileParams(CoreModel):
         Field(
             description=(
                 "The existing reservation to use for instance provisioning."
-                " Supports AWS Capacity Reservations and Capacity Blocks"
+                " Supports AWS Capacity Reservations, AWS Capacity Blocks, and GCP reservations"
             )
         ),
     ] = None

dstack/_internal/core/models/runs.py

@@ -462,11 +462,12 @@ class RunSpec(generate_dual_core_model(RunSpecConfig)):
     configuration: Annotated[AnyRunConfiguration, Field(discriminator="type")]
     profile: Annotated[Optional[Profile], Field(description="The profile parameters")] = None
     ssh_key_pub: Annotated[
-        str,
+        Optional[str],
         Field(
             description="The contents of the SSH public key that will be used to connect to the run."
+            " Can be empty only before the run is submitted."
         ),
-    ]
+    ] = None
     # merged_profile stores profile parameters merged from profile and configuration.
     # Read profile parameters from merged_profile instead of profile directly.
     # TODO: make merged_profile a computed field after migrating to pydanticV2
@@ -552,6 +553,7 @@ class Run(CoreModel):
     deployment_num: int = 0  # default for compatibility with pre-0.19.14 servers
     error: Optional[str] = None
     deleted: Optional[bool] = None
+    next_triggered_at: Optional[datetime] = None
 
     def is_deployment_in_progress(self) -> bool:
         return any(

dstack/_internal/core/models/users.py

@@ -30,6 +30,7 @@ class User(CoreModel):
     email: Optional[str]
     active: bool
     permissions: UserPermissions
+    ssh_public_key: Optional[str] = None
 
 
 class UserTokenCreds(CoreModel):
@@ -38,3 +39,12 @@ class UserTokenCreds(CoreModel):
 
 class UserWithCreds(User):
     creds: UserTokenCreds
+    ssh_private_key: Optional[str] = None
+
+
+class UserHookConfig(CoreModel):
+    """
+    This class can be inherited to extend the user creation configuration passed to the hooks.
+    """
+
+    pass

dstack/_internal/core/services/configs/__init__.py

@@ -117,6 +117,7 @@ class ConfigManager:
 
     @property
     def dstack_key_path(self) -> Path:
+        # TODO: Remove since 0.19.40
         return self.dstack_ssh_dir / "id_rsa"
 
     @property

dstack/_internal/core/services/ssh/key_manager.py

@@ -0,0 +1,56 @@
+import os
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import TYPE_CHECKING, Optional
+
+from dstack._internal.core.models.users import UserWithCreds
+
+if TYPE_CHECKING:
+    from dstack.api.server import APIClient
+
+KEY_REFRESH_RATE = timedelta(minutes=10)  # redownload the key periodically in case it was rotated
+
+
+@dataclass
+class UserSSHKey:
+    public_key: str
+    private_key_path: Path
+
+
+class UserSSHKeyManager:
+    def __init__(self, api_client: "APIClient", ssh_keys_dir: Path) -> None:
+        self._api_client = api_client
+        self._key_path = ssh_keys_dir / api_client.get_token_hash()
+        self._pub_key_path = self._key_path.with_suffix(".pub")
+
+    def get_user_key(self) -> Optional[UserSSHKey]:
+        """
+        Return the up-to-date user key, or None if the user has no key (if created before 0.19.33)
+        """
+        if (
+            not self._key_path.exists()
+            or not self._pub_key_path.exists()
+            or datetime.now() - datetime.fromtimestamp(self._key_path.stat().st_mtime)
+            > KEY_REFRESH_RATE
+        ):
+            if not self._download_user_key():
+                return None
+        return UserSSHKey(
+            public_key=self._pub_key_path.read_text(), private_key_path=self._key_path
+        )
+
+    def _download_user_key(self) -> bool:
+        user = self._api_client.users.get_my_user()
+        if not (isinstance(user, UserWithCreds) and user.ssh_public_key and user.ssh_private_key):
+            return False
+
+        def key_opener(path, flags):
+            return os.open(path, flags, 0o600)
+
+        with open(self._key_path, "w", opener=key_opener) as f:
+            f.write(user.ssh_private_key)
+        with open(self._pub_key_path, "w") as f:
+            f.write(user.ssh_public_key)
+
+        return True

dstack/_internal/server/background/tasks/process_instances.py

@@ -558,10 +558,14 @@ async def _create_instance(session: AsyncSession, instance: InstanceModel) -> No
         if (
             _is_fleet_master_instance(instance)
             and instance_offer.backend in BACKENDS_WITH_PLACEMENT_GROUPS_SUPPORT
+            and isinstance(compute, ComputeWithPlacementGroupSupport)
+            and (
+                compute.are_placement_groups_compatible_with_reservations(instance_offer.backend)
+                or instance_configuration.reservation is None
+            )
             and instance.fleet
             and _is_cloud_cluster(instance.fleet)
         ):
-            assert isinstance(compute, ComputeWithPlacementGroupSupport)
             placement_group_model = _find_suitable_placement_group(
                 placement_groups=placement_group_models,
                 instance_offer=instance_offer,

dstack/_internal/server/background/tasks/process_running_jobs.py

@@ -243,6 +243,7 @@ async def _process_running_job(session: AsyncSession, job_model: JobModel):
                 job_submission.age,
             )
             ssh_user = job_provisioning_data.username
+            assert run.run_spec.ssh_key_pub is not None
             user_ssh_key = run.run_spec.ssh_key_pub.strip()
             public_keys = [project.ssh_public_key.strip(), user_ssh_key]
             if job_provisioning_data.backend == BackendType.LOCAL:

dstack/_internal/server/migrations/versions/ff1d94f65b08_user_ssh_key.py

@@ -0,0 +1,34 @@
+"""user.ssh_key
+
+Revision ID: ff1d94f65b08
+Revises: 2498ab323443
+Create Date: 2025-10-09 20:31:31.166786
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "ff1d94f65b08"
+down_revision = "2498ab323443"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("users", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("ssh_private_key", sa.Text(), nullable=True))
+        batch_op.add_column(sa.Column("ssh_public_key", sa.Text(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("users", schema=None) as batch_op:
+        batch_op.drop_column("ssh_public_key")
+        batch_op.drop_column("ssh_private_key")
+
+    # ### end Alembic commands ###

dstack/_internal/server/models.py

@@ -190,6 +190,12 @@ class UserModel(BaseModel):
     # deactivated users cannot access API
     active: Mapped[bool] = mapped_column(Boolean, default=True)
 
+    # SSH keys can be null for users created before 0.19.33.
+    # Keys for those users are being gradually generated on /get_my_user calls.
+    # TODO: make keys required in a future version.
+    ssh_private_key: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+    ssh_public_key: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+
     email: Mapped[Optional[str]] = mapped_column(String(200), nullable=True)
 
     projects_quota: Mapped[int] = mapped_column(

dstack/_internal/server/routers/metrics.py

@@ -1,5 +1,6 @@
 from datetime import datetime
 from typing import Optional, Tuple
+from uuid import UUID
 
 from fastapi import APIRouter, Depends
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -29,6 +30,7 @@ router = APIRouter(
 )
 async def get_job_metrics(
     run_name: str,
+    run_id: Optional[UUID] = None,
     replica_num: int = 0,
     job_num: int = 0,
     limit: int = 1,
@@ -39,8 +41,9 @@ async def get_job_metrics(
 ):
     """
     Returns job-level metrics such as hardware utilization
-    given `run_name`, `replica_num`, and `job_num`.
-    If only `run_name` is specified, returns metrics of `(replica_num=0, job_num=0)`.
+    given `run_name`, `run_id`, `replica_num`, and `job_num`.
+    If only `run_name` is specified, returns metrics of `(replica_num=0, job_num=0)`
+    of the latest run with the given name.
     By default, returns one latest sample. To control time window/number of samples, use
     `limit`, `after`, `before`.
 
@@ -61,6 +64,7 @@ async def get_job_metrics(
         session=session,
         project=project,
         run_name=run_name,
+        run_id=run_id,
         replica_num=replica_num,
         job_num=job_num,
     )

dstack/_internal/server/routers/runs.py

@@ -17,7 +17,7 @@ from dstack._internal.server.schemas.runs import (
     SubmitRunRequest,
 )
 from dstack._internal.server.security.permissions import Authenticated, ProjectMember
-from dstack._internal.server.services import runs
+from dstack._internal.server.services import runs, users
 from dstack._internal.server.utils.routers import (
     CustomORJSONResponse,
     get_base_api_additional_responses,
@@ -111,6 +111,8 @@ async def get_plan(
     This is an optional step before calling `/apply`.
     """
     user, project = user_project
+    if not user.ssh_public_key and not body.run_spec.ssh_key_pub:
+        await users.refresh_ssh_key(session=session, user=user, username=user.name)
     run_plan = await runs.get_plan(
         session=session,
         project=project,
@@ -137,6 +139,8 @@ async def apply_plan(
     If the existing run is active and cannot be updated, it must be stopped first.
     """
     user, project = user_project
+    if not user.ssh_public_key and not body.plan.run_spec.ssh_key_pub:
+        await users.refresh_ssh_key(session=session, user=user, username=user.name)
     return CustomORJSONResponse(
         await runs.apply_plan(
             session=session,

dstack/_internal/server/routers/users.py

@@ -36,11 +36,18 @@ async def list_users(
     return CustomORJSONResponse(await users.list_users_for_user(session=session, user=user))
 
 
-@router.post("/get_my_user", response_model=User)
+@router.post("/get_my_user", response_model=UserWithCreds)
 async def get_my_user(
+    session: AsyncSession = Depends(get_session),
     user: UserModel = Depends(Authenticated()),
 ):
-    return CustomORJSONResponse(users.user_model_to_user(user))
+    if user.ssh_private_key is None or user.ssh_public_key is None:
+        # Generate keys for pre-0.19.33 users
+        updated_user = await users.refresh_ssh_key(session=session, user=user, username=user.name)
+        if updated_user is None:
+            raise ResourceNotExistsError()
+        user = updated_user
+    return CustomORJSONResponse(users.user_model_to_user_with_creds(user))
 
 
 @router.post("/get_user", response_model=UserWithCreds)
@@ -91,6 +98,18 @@ async def update_user(
     return CustomORJSONResponse(users.user_model_to_user(res))
 
 
+@router.post("/refresh_ssh_key", response_model=UserWithCreds)
+async def refresh_ssh_key(
+    body: RefreshTokenRequest,
+    session: AsyncSession = Depends(get_session),
+    user: UserModel = Depends(Authenticated()),
+):
+    res = await users.refresh_ssh_key(session=session, user=user, username=body.username)
+    if res is None:
+        raise ResourceNotExistsError()
+    return CustomORJSONResponse(users.user_model_to_user_with_creds(res))
+
+
 @router.post("/refresh_token", response_model=UserWithCreds)
 async def refresh_token(
     body: RefreshTokenRequest,

dstack/_internal/server/services/jobs/__init__.py

@@ -97,19 +97,28 @@ def find_job(jobs: List[Job], replica_num: int, job_num: int) -> Job:
 
 
 async def get_run_job_model(
-    session: AsyncSession, project: ProjectModel, run_name: str, replica_num: int, job_num: int
+    session: AsyncSession,
+    project: ProjectModel,
+    run_name: str,
+    run_id: Optional[UUID],
+    replica_num: int,
+    job_num: int,
 ) -> Optional[JobModel]:
+    filters = [
+        RunModel.project_id == project.id,
+        RunModel.run_name == run_name,
+        JobModel.replica_num == replica_num,
+        JobModel.job_num == job_num,
+    ]
+    if run_id is not None:
+        filters.append(RunModel.id == run_id)
+    else:
+        # Assuming run_name is unique for non-deleted runs
+        filters.append(RunModel.deleted == False)
     res = await session.execute(
         select(JobModel)
         .join(JobModel.run)
-        .where(
-            RunModel.project_id == project.id,
-            # assuming run_name is unique for non-deleted runs
-            RunModel.run_name == run_name,
-            RunModel.deleted == False,
-            JobModel.replica_num == replica_num,
-            JobModel.job_num == job_num,
-        )
+        .where(*filters)
         .order_by(JobModel.submission_num.desc())
         .limit(1)
     )

dstack/_internal/server/services/offers.py

@@ -215,6 +215,7 @@ def generate_shared_offer(
         ),
         region=offer.region,
         price=offer.price,
+        backend_data=offer.backend_data,
         availability=offer.availability,
         blocks=blocks,
         total_blocks=total_blocks,

dstack/_internal/server/services/runs.py

@@ -317,7 +317,7 @@ async def get_plan(
         spec=effective_run_spec,
     )
     effective_run_spec = RunSpec.parse_obj(effective_run_spec.dict())
-    _validate_run_spec_and_set_defaults(effective_run_spec)
+    _validate_run_spec_and_set_defaults(user, effective_run_spec)
 
     profile = effective_run_spec.merged_profile
     creation_policy = profile.creation_policy
@@ -422,7 +422,7 @@ async def apply_plan(
     )
     # Spec must be copied by parsing to calculate merged_profile
     run_spec = RunSpec.parse_obj(run_spec.dict())
-    _validate_run_spec_and_set_defaults(run_spec)
+    _validate_run_spec_and_set_defaults(user, run_spec)
     if run_spec.run_name is None:
         return await submit_run(
             session=session,
@@ -489,7 +489,7 @@ async def submit_run(
     project: ProjectModel,
     run_spec: RunSpec,
 ) -> Run:
-    _validate_run_spec_and_set_defaults(run_spec)
+    _validate_run_spec_and_set_defaults(user, run_spec)
     repo = await _get_run_repo_or_error(
         session=session,
         project=project,
@@ -715,6 +715,9 @@ def run_model_to_run(
     status_message = _get_run_status_message(run_model)
     error = _get_run_error(run_model)
     fleet = _get_run_fleet(run_model)
+    next_triggered_at = None
+    if not run_model.status.is_finished():
+        next_triggered_at = _get_next_triggered_at(run_spec)
     run = Run(
         id=run_model.id,
         project_name=run_model.project.name,
@@ -734,6 +737,7 @@ def run_model_to_run(
         deployment_num=run_model.deployment_num,
         error=error,
         deleted=run_model.deleted,
+        next_triggered_at=next_triggered_at,
     )
     run.cost = _get_run_cost(run)
     return run
@@ -981,7 +985,7 @@ def _get_job_submission_cost(job_submission: JobSubmission) -> float:
     return job_submission.job_provisioning_data.price * duration_hours
 
 
-def _validate_run_spec_and_set_defaults(run_spec: RunSpec):
+def _validate_run_spec_and_set_defaults(user: UserModel, run_spec: RunSpec):
     # This function may set defaults for null run_spec values,
     # although most defaults are resolved when building job_spec
     # so that we can keep both the original user-supplied value (null in run_spec)
@@ -1031,6 +1035,11 @@ def _validate_run_spec_and_set_defaults(run_spec: RunSpec):
     if run_spec.configuration.priority is None:
         run_spec.configuration.priority = RUN_PRIORITY_DEFAULT
     set_resources_defaults(run_spec.configuration.resources)
+    if run_spec.ssh_key_pub is None:
+        if user.ssh_public_key:
+            run_spec.ssh_key_pub = user.ssh_public_key
+        else:
+            raise ServerClientError("ssh_key_pub must be set if the user has no ssh_public_key")
 
 
 _UPDATABLE_SPEC_FIELDS = ["configuration_path", "configuration"]

dstack/_internal/server/services/users.py

@@ -12,6 +12,7 @@ from dstack._internal.core.errors import ResourceExistsError, ServerClientError
 from dstack._internal.core.models.users import (
     GlobalRole,
     User,
+    UserHookConfig,
     UserPermissions,
     UserTokenCreds,
     UserWithCreds,
@@ -19,6 +20,8 @@ from dstack._internal.core.models.users import (
 from dstack._internal.server.models import DecryptedString, UserModel
 from dstack._internal.server.services.permissions import get_default_permissions
 from dstack._internal.server.utils.routers import error_forbidden
+from dstack._internal.utils import crypto
+from dstack._internal.utils.common import run_async
 from dstack._internal.utils.logging import get_logger
 
 logger = get_logger(__name__)
@@ -77,6 +80,7 @@ async def create_user(
     email: Optional[str] = None,
     active: bool = True,
     token: Optional[str] = None,
+    config: Optional[UserHookConfig] = None,
 ) -> UserModel:
     validate_username(username)
     user_model = await get_user_model_by_name(session=session, username=username, ignore_case=True)
@@ -84,6 +88,7 @@ async def create_user(
         raise ResourceExistsError()
     if token is None:
         token = str(uuid.uuid4())
+    private_bytes, public_bytes = await run_async(crypto.generate_rsa_key_pair_bytes, username)
     user = UserModel(
         id=uuid.uuid4(),
         name=username,
@@ -92,11 +97,13 @@ async def create_user(
         token_hash=get_token_hash(token),
         email=email,
         active=active,
+        ssh_private_key=private_bytes.decode(),
+        ssh_public_key=public_bytes.decode(),
     )
     session.add(user)
     await session.commit()
     for func in _CREATE_USER_HOOKS:
-        await func(session, user)
+        await func(session, user, config)
     return user
 
 
@@ -120,6 +127,27 @@ async def update_user(
     return await get_user_model_by_name_or_error(session=session, username=username)
 
 
+async def refresh_ssh_key(
+    session: AsyncSession,
+    user: UserModel,
+    username: str,
+) -> Optional[UserModel]:
+    logger.debug("Refreshing SSH key for user [code]%s[/code]", username)
+    if user.global_role != GlobalRole.ADMIN and user.name != username:
+        raise error_forbidden()
+    private_bytes, public_bytes = await run_async(crypto.generate_rsa_key_pair_bytes, username)
+    await session.execute(
+        update(UserModel)
+        .where(UserModel.name == username)
+        .values(
+            ssh_private_key=private_bytes.decode(),
+            ssh_public_key=public_bytes.decode(),
+        )
+    )
+    await session.commit()
+    return await get_user_model_by_name(session=session, username=username)
+
+
 async def refresh_user_token(
     session: AsyncSession,
     user: UserModel,
@@ -199,6 +227,7 @@ def user_model_to_user(user_model: UserModel) -> User:
         email=user_model.email,
         active=user_model.active,
         permissions=get_user_permissions(user_model),
+        ssh_public_key=user_model.ssh_public_key,
     )
 
 
@@ -211,7 +240,9 @@ def user_model_to_user_with_creds(user_model: UserModel) -> UserWithCreds:
         email=user_model.email,
         active=user_model.active,
         permissions=get_user_permissions(user_model),
+        ssh_public_key=user_model.ssh_public_key,
         creds=UserTokenCreds(token=user_model.token.get_plaintext_or_error()),
+        ssh_private_key=user_model.ssh_private_key,
     )
 
 
@@ -238,7 +269,9 @@ def is_valid_username(username: str) -> bool:
 _CREATE_USER_HOOKS = []
 
 
-def register_create_user_hook(func: Callable[[AsyncSession, UserModel], Awaitable[None]]):
+def register_create_user_hook(
+    func: Callable[[AsyncSession, UserModel, Optional[UserHookConfig]], Awaitable[None]],
+):
     _CREATE_USER_HOOKS.append(func)
 
 

dstack/_internal/server/statics/index.html

@@ -1,3 +1,3 @@
 <!doctype html><html lang="en"><head><meta charset="utf-8"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><title>dstack</title><meta name="description" content="Get GPUs at the best prices and availability from a wide range of providers. No cloud account of your own is required.
 "/><link rel="preconnect" href="https://fonts.googleapis.com"><link rel="preconnect" href="https://fonts.gstatic.com" crossorigin><link href="https://fonts.googleapis.com/css2?family=Roboto:ital,wght@0,100;0,300;0,400;0,500;0,700;0,900;1,100;1,300;1,400;1,500;1,700;1,900&display=swap" rel="stylesheet"><meta name="og:title" content="dstack"><meta name="og:type" content="article"><meta name="og:image" content="/splash_thumbnail.png"><meta name="og:description" content="Get GPUs at the best prices and availability from a wide range of providers. No cloud account of your own is required.
-"><link rel="icon" type="image/x-icon" href="/assets/favicon.ico"><link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png"><link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png"><link rel="icon" type="image/png" sizes="48x48" href="/assets/favicon-48x48.png"><link rel="manifest" href="/assets/manifest.webmanifest"><meta name="mobile-web-app-capable" content="yes"><meta name="theme-color" content="#fff"><meta name="application-name" content="dstackai"><link rel="apple-touch-icon" sizes="57x57" href="/assets/apple-touch-icon-57x57.png"><link rel="apple-touch-icon" sizes="60x60" href="/assets/apple-touch-icon-60x60.png"><link rel="apple-touch-icon" sizes="72x72" href="/assets/apple-touch-icon-72x72.png"><link rel="apple-touch-icon" sizes="76x76" href="/assets/apple-touch-icon-76x76.png"><link rel="apple-touch-icon" sizes="114x114" href="/assets/apple-touch-icon-114x114.png"><link rel="apple-touch-icon" sizes="120x120" href="/assets/apple-touch-icon-120x120.png"><link rel="apple-touch-icon" sizes="144x144" href="/assets/apple-touch-icon-144x144.png"><link rel="apple-touch-icon" sizes="152x152" href="/assets/apple-touch-icon-152x152.png"><link rel="apple-touch-icon" sizes="167x167" href="/assets/apple-touch-icon-167x167.png"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon-180x180.png"><link rel="apple-touch-icon" sizes="1024x1024" href="/assets/apple-touch-icon-1024x1024.png"><meta name="apple-mobile-web-app-capable" content="yes"><meta name="apple-mobile-web-app-status-bar-style" content="black-translucent"><meta name="apple-mobile-web-app-title" content="dstackai"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-640x1136.png"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1136x640.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-750x1334.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1334x750.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1125x2436.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2436x1125.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1170x2532.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2532x1170.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1179x2556.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2556x1179.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-828x1792.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1792x828.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2688.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2688x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2208.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2208x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1284x2778.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2778x1284.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1290x2796.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2796x1290.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1488x2266.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2266x1488.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1536x2048.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2048x1536.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1620x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1620.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1640x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1640.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2388.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2388x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2224.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2224x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-2048x2732.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2732x2048.png"><meta name="msapplication-TileColor" content="#fff"><meta name="msapplication-TileImage" content="/assets/mstile-144x144.png"><meta name="msapplication-config" content="/assets/browserconfig.xml"><link rel="yandex-tableau-widget" href="/assets/yandex-browser-manifest.json"><script defer="defer" src="/main-c51afa7f243e24d3e446.js"></script><link href="/main-56191fbfe77f49b251de.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div class="b-page-header" id="header"></div><div id="root"></div><script async src="https://widget.kapa.ai/kapa-widget.bundle.js" data-website-id="11a9339d-20ce-4ddb-9ba3-1b6e29afe8eb" data-project-name="dstack" data-project-color="rgba(0, 0, 0, 0.87)" data-font-size-lg="0.78rem" data-button-hide="true" data-modal-image="/logo-notext.svg" data-modal-z-index="1100" data-modal-title="Ask me anything" data-font-family='metro-web, Metro, -apple-system, "system-ui", "Segoe UI", Roboto' data-project-logo="/assets/images/kapa.svg" data-modal-disclaimer="This is a custom LLM for dstack with access to Documentation, API references and GitHub issues. This feature is experimental - Give it a try!" data-user-analytics-fingerprint-enabled="true"></script></body></html>
+"><link rel="icon" type="image/x-icon" href="/assets/favicon.ico"><link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png"><link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png"><link rel="icon" type="image/png" sizes="48x48" href="/assets/favicon-48x48.png"><link rel="manifest" href="/assets/manifest.webmanifest"><meta name="mobile-web-app-capable" content="yes"><meta name="theme-color" content="#fff"><meta name="application-name" content="dstackai"><link rel="apple-touch-icon" sizes="57x57" href="/assets/apple-touch-icon-57x57.png"><link rel="apple-touch-icon" sizes="60x60" href="/assets/apple-touch-icon-60x60.png"><link rel="apple-touch-icon" sizes="72x72" href="/assets/apple-touch-icon-72x72.png"><link rel="apple-touch-icon" sizes="76x76" href="/assets/apple-touch-icon-76x76.png"><link rel="apple-touch-icon" sizes="114x114" href="/assets/apple-touch-icon-114x114.png"><link rel="apple-touch-icon" sizes="120x120" href="/assets/apple-touch-icon-120x120.png"><link rel="apple-touch-icon" sizes="144x144" href="/assets/apple-touch-icon-144x144.png"><link rel="apple-touch-icon" sizes="152x152" href="/assets/apple-touch-icon-152x152.png"><link rel="apple-touch-icon" sizes="167x167" href="/assets/apple-touch-icon-167x167.png"><link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon-180x180.png"><link rel="apple-touch-icon" sizes="1024x1024" href="/assets/apple-touch-icon-1024x1024.png"><meta name="apple-mobile-web-app-capable" content="yes"><meta name="apple-mobile-web-app-status-bar-style" content="black-translucent"><meta name="apple-mobile-web-app-title" content="dstackai"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-640x1136.png"><link rel="apple-touch-startup-image" media="(device-width: 320px) and (device-height: 568px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1136x640.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-750x1334.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 667px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1334x750.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1125x2436.png"><link rel="apple-touch-startup-image" media="(device-width: 375px) and (device-height: 812px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2436x1125.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1170x2532.png"><link rel="apple-touch-startup-image" media="(device-width: 390px) and (device-height: 844px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2532x1170.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1179x2556.png"><link rel="apple-touch-startup-image" media="(device-width: 393px) and (device-height: 852px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2556x1179.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-828x1792.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-1792x828.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2688.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 896px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2688x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1242x2208.png"><link rel="apple-touch-startup-image" media="(device-width: 414px) and (device-height: 736px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2208x1242.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1284x2778.png"><link rel="apple-touch-startup-image" media="(device-width: 428px) and (device-height: 926px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2778x1284.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1290x2796.png"><link rel="apple-touch-startup-image" media="(device-width: 430px) and (device-height: 932px) and (-webkit-device-pixel-ratio: 3) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2796x1290.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1488x2266.png"><link rel="apple-touch-startup-image" media="(device-width: 744px) and (device-height: 1133px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2266x1488.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1536x2048.png"><link rel="apple-touch-startup-image" media="(device-width: 768px) and (device-height: 1024px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2048x1536.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1620x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 810px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1620.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1640x2160.png"><link rel="apple-touch-startup-image" media="(device-width: 820px) and (device-height: 1080px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2160x1640.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2388.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1194px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2388x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-1668x2224.png"><link rel="apple-touch-startup-image" media="(device-width: 834px) and (device-height: 1112px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2224x1668.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: portrait)" href="/assets/apple-touch-startup-image-2048x2732.png"><link rel="apple-touch-startup-image" media="(device-width: 1024px) and (device-height: 1366px) and (-webkit-device-pixel-ratio: 2) and (orientation: landscape)" href="/assets/apple-touch-startup-image-2732x2048.png"><meta name="msapplication-TileColor" content="#fff"><meta name="msapplication-TileImage" content="/assets/mstile-144x144.png"><meta name="msapplication-config" content="/assets/browserconfig.xml"><link rel="yandex-tableau-widget" href="/assets/yandex-browser-manifest.json"><script defer="defer" src="/main-e79754c136f1d8e4e7e6.js"></script><link href="/main-720ce3a11140daa480cc.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div class="b-page-header" id="header"></div><div id="root"></div><script async src="https://widget.kapa.ai/kapa-widget.bundle.js" data-website-id="11a9339d-20ce-4ddb-9ba3-1b6e29afe8eb" data-project-name="dstack" data-project-color="rgba(0, 0, 0, 0.87)" data-font-size-lg="0.78rem" data-button-hide="true" data-modal-image="/logo-notext.svg" data-modal-z-index="1100" data-modal-title="Ask me anything" data-font-family='metro-web, Metro, -apple-system, "system-ui", "Segoe UI", Roboto' data-project-logo="/assets/images/kapa.svg" data-modal-disclaimer="This is a custom LLM for dstack with access to Documentation, API references and GitHub issues. This feature is experimental - Give it a try!" data-user-analytics-fingerprint-enabled="true"></script></body></html>