dstack 0.19.33__tar.gz → 0.19.34__tar.gz

{dstack-0.19.33 → dstack-0.19.34}/CONTRIBUTING.md

@@ -2,6 +2,19 @@
 
 We appreciate your interest in contributing to `dstack`! This document will help you get up to speed with `dstack` codebase and guide you through the contribution process.
 
+## AI Assistance Notice
+
+If you are using any kind of AI assistance while contributing to `dstack`,
+**this must be disclosed in the pull request**, along with the extent to
+which AI assistance was used.
+As an exception, tab-completions and trivial PRs don't need to be disclosed.
+
+An example disclosure:
+
+> This PR was written primarily by Claude Code.
+
+Failure to disclose this, makes it difficult to determine how much scrutiny to apply to the contribution. Please be respectful to maintainers and disclose AI assistance.
+
 ## Set up your development environment
 
 Follow [contributing/DEVELOPMENT.md](contributing/DEVELOPMENT.md).

{dstack-0.19.33 → dstack-0.19.34}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dstack
-Version: 0.19.33
+Version: 0.19.34
 Summary: dstack is an open-source orchestration engine for running AI workloads on any cloud or on-premises.
 Project-URL: Homepage, https://dstack.ai
 Project-URL: Source, https://github.com/dstackai/dstack
@@ -22,7 +22,7 @@ Requires-Dist: cryptography
 Requires-Dist: cursor
 Requires-Dist: filelock
 Requires-Dist: gitpython
-Requires-Dist: gpuhunt==0.1.10
+Requires-Dist: gpuhunt==0.1.11
 Requires-Dist: ignore-python>=0.2.0
 Requires-Dist: jsonschema
 Requires-Dist: orjson

{dstack-0.19.33 → dstack-0.19.34}/docs/docs/concepts/backends.md

@@ -1047,6 +1047,31 @@ projects:
     
     If on-demand provisioning is important, we recommend using [VM-based](#vm-based) backends as they already support auto-scaling. -->
 
+??? info "Required permissions"
+    The following Kubernetes permissions are sufficient for `dstack` to work:
+
+    ```yaml
+    apiVersion: rbac.authorization.k8s.io/v1
+    kind: ClusterRole
+    metadata:
+      name: dstack-backend
+    rules:
+    - apiGroups: [""]
+      resources: ["namespaces"]
+      verbs: ["get", "create"]
+    - apiGroups: [""]
+      resources: ["pods"]
+      verbs: ["get", "create", "delete"]
+    - apiGroups: [""]
+      resources: ["services"]
+      verbs: ["get", "create", "delete"]
+    - apiGroups: [""]
+      resources: ["nodes"]
+      verbs: ["list"]
+    ```
+    
+    Ensure you've created a ClusterRoleBinding to grant the role to the user or the service account you're using.
+
 > To learn more, see the [Kubernetes](../guides/kubernetes.md) guide.
 
 ### RunPod

{dstack-0.19.33 → dstack-0.19.34}/frontend/src/locale/en.json

@@ -417,6 +417,8 @@
       "backend": "Backend",
       "region": "Region",
       "instance_id": "Instance ID",
+      "schedule": "Schedule",
+      "next_run": "Next run",
       "resources": "Resources",
       "spot": "Spot",
       "termination_reason": "Termination reason",

{dstack-0.19.33 → dstack-0.19.34}/frontend/src/pages/Runs/Details/Jobs/Details/JobDetails/index.tsx

@@ -9,6 +9,7 @@ import { useGetRunQuery } from 'services/run';
 
 import {
     getJobError,
+    getJobFinishedAt,
     getJobListItemBackend,
     getJobListItemInstance,
     getJobListItemPrice,
@@ -59,6 +60,11 @@ export const JobDetails = () => {
                         <div>{getJobSubmittedAt(jobData)}</div>
                     </div>
 
+                    <div>
+                        <Box variant="awsui-key-label">{t('projects.run.finished_at')}</Box>
+                        <div>{getJobFinishedAt(jobData)}</div>
+                    </div>
+
                     <div>
                         <Box variant="awsui-key-label">{t('projects.run.status')}</Box>
                         <div>

{dstack-0.19.33 → dstack-0.19.34}/frontend/src/pages/Runs/Details/Jobs/List/helpers.ts

@@ -39,6 +39,11 @@ export const getJobSubmittedAt = (job: IJob) => {
         : '';
 };
 
+export const getJobFinishedAt = (job: IJob) => {
+    const finished_at = job.job_submissions?.[job.job_submissions.length - 1].finished_at;
+    return finished_at ? format(new Date(finished_at), DATE_TIME_FORMAT) : '';
+};
+
 export const getJobStatus = (job: IJob) => {
     return job.job_submissions?.[job.job_submissions.length - 1].status;
 };

{dstack-0.19.33 → dstack-0.19.34}/frontend/src/pages/Runs/Details/Jobs/List/hooks.tsx

@@ -8,6 +8,7 @@ import { ROUTES } from 'routes';
 
 import {
     getJobError,
+    getJobFinishedAt,
     getJobListItemBackend,
     getJobListItemInstance,
     getJobListItemPrice,
@@ -48,6 +49,11 @@ export const useColumnsDefinitions = ({
             header: t('projects.run.submitted_at'),
             cell: getJobSubmittedAt,
         },
+        {
+            id: 'finished_at',
+            header: t('projects.run.finished_at'),
+            cell: getJobFinishedAt,
+        },
         {
             id: 'status',
             header: t('projects.run.status'),

{dstack-0.19.33 → dstack-0.19.34}/frontend/src/pages/Runs/Details/Jobs/Metrics/index.tsx

@@ -35,6 +35,7 @@ export const JobMetrics: React.FC = () => {
     const { cpuChartProps, memoryChartProps, eachGPUChartProps, eachGPUMemoryChartProps, isLoading } = useMetricsData({
         project_name: paramProjectName,
         run_name: runData?.run_spec.run_name ?? '',
+        run_id: runData?.id ?? '',
         job_num: jobData?.job_spec.job_num ?? 0,
         limit: 1000,
     });

{dstack-0.19.33 → dstack-0.19.34}/frontend/src/pages/Runs/Details/RunDetails/index.tsx

@@ -15,11 +15,13 @@ import { finishedRunStatuses } from 'pages/Runs/constants';
 import { runIsStopped } from 'pages/Runs/utils';
 
 import {
+    getRunListFinishedAt,
     getRunListItemBackend,
     getRunListItemInstanceId,
     getRunListItemPrice,
     getRunListItemRegion,
     getRunListItemResources,
+    getRunListItemSchedule,
     getRunListItemServiceUrl,
     getRunListItemSpot,
 } from '../../List/helpers';
@@ -38,6 +40,8 @@ export const RunDetails = () => {
     });
 
     const serviceUrl = runData ? getRunListItemServiceUrl(runData) : null;
+    const schedule = runData ? getRunListItemSchedule(runData) : null;
+    const nextTriggeredAt = runData ? runData.next_triggered_at : null;
 
     if (isLoadingRun)
         return (
@@ -56,6 +60,8 @@ export const RunDetails = () => {
         ? runData.latest_job_submission?.termination_reason
         : null;
 
+    const finishedAt = getRunListFinishedAt(runData);
+
     return (
         <>
             <Container header={<Header variant="h2">{t('common.general')}</Header>}>
@@ -98,7 +104,7 @@ export const RunDetails = () => {
 
                     <div>
                         <Box variant="awsui-key-label">{t('projects.run.finished_at')}</Box>
-                        <div>{runData.terminated_at ? format(new Date(runData.terminated_at), DATE_TIME_FORMAT) : '-'}</div>
+                        <div>{finishedAt ? format(new Date(finishedAt), DATE_TIME_FORMAT) : '-'}</div>
                     </div>
 
                     <div>
@@ -115,7 +121,7 @@ export const RunDetails = () => {
 
                     <div>
                         <Box variant="awsui-key-label">{t('projects.run.error')}</Box>
-                        <div>{getRunError(runData)}</div>
+                        <div>{getRunError(runData) ?? '-'}</div>
                     </div>
 
                     <div>
@@ -138,6 +144,11 @@ export const RunDetails = () => {
                         <div>{getRunListItemResources(runData)}</div>
                     </div>
 
+                    <div>
+                        <Box variant="awsui-key-label">{t('projects.run.backend')}</Box>
+                        <div>{getRunListItemBackend(runData)}</div>
+                    </div>
+
                     <div>
                         <Box variant="awsui-key-label">{t('projects.run.region')}</Box>
                         <div>{getRunListItemRegion(runData)}</div>
@@ -152,11 +163,6 @@ export const RunDetails = () => {
                         <Box variant="awsui-key-label">{t('projects.run.spot')}</Box>
                         <div>{getRunListItemSpot(runData)}</div>
                     </div>
-
-                    <div>
-                        <Box variant="awsui-key-label">{t('projects.run.backend')}</Box>
-                        <div>{getRunListItemBackend(runData)}</div>
-                    </div>
                 </ColumnLayout>
 
                 {serviceUrl && (
@@ -169,6 +175,19 @@ export const RunDetails = () => {
                         </div>
                     </ColumnLayout>
                 )}
+
+                {schedule && (
+                    <ColumnLayout columns={4} variant="text-grid">
+                        <div>
+                            <Box variant="awsui-key-label">{t('projects.run.schedule')}</Box>
+                            <div>{schedule}</div>
+                        </div>
+                        <div>
+                            <Box variant="awsui-key-label">{t('projects.run.next_run')}</Box>
+                            <div>{nextTriggeredAt ? format(new Date(nextTriggeredAt), DATE_TIME_FORMAT) : '-'}</div>
+                        </div>
+                    </ColumnLayout>
+                )}
             </Container>
 
             {runData.run_spec.configuration.type === 'dev-environment' && !runIsStopped(runData.status) && (

{dstack-0.19.33 → dstack-0.19.34}/frontend/src/pages/Runs/List/Preferences/consts.ts

@@ -9,12 +9,12 @@ export const DEFAULT_PREFERENCES: CollectionPreferencesProps.Preferences = {
         { id: 'hub_user_name', visible: true },
         { id: 'price', visible: true },
         { id: 'submitted_at', visible: true },
+        { id: 'finished_at', visible: true },
         { id: 'status', visible: true },
         { id: 'error', visible: true },
         { id: 'cost', visible: true },
         // hidden by default
         { id: 'priority', visible: false },
-        { id: 'finished_at', visible: false },
         { id: 'project', visible: false },
         { id: 'repo', visible: false },
         { id: 'instance', visible: false },

{dstack-0.19.33 → dstack-0.19.34}/frontend/src/pages/Runs/List/helpers.ts

@@ -2,7 +2,7 @@ import { groupBy as _groupBy } from 'lodash';
 
 import { getBaseUrl } from 'App/helpers';
 
-import { finishedJobs } from '../constants';
+import { finishedJobs, finishedRunStatuses } from '../constants';
 import { getJobStatus } from '../Details/Jobs/List/helpers';
 
 export const getGroupedRunsByProjectAndRepoID = (runs: IRun[]) => {
@@ -14,7 +14,7 @@ export const getRunListItemResources = (run: IRun) => {
         return '-';
     }
 
-    return run.latest_job_submission?.job_provisioning_data?.instance_type?.resources?.description;
+    return run.latest_job_submission?.job_provisioning_data?.instance_type?.resources?.description ?? '-';
 };
 
 export const getRunListItemSpotLabelKey = (run: IRun) => {
@@ -31,7 +31,7 @@ export const getRunListItemSpotLabelKey = (run: IRun) => {
 
 export const getRunListItemSpot = (run: IRun) => {
     if (run.jobs.length > 1) {
-        return '';
+        return '-';
     }
 
     return run.latest_job_submission?.job_provisioning_data?.instance_type?.resources?.spot?.toString() ?? '-';
@@ -57,7 +57,7 @@ export const getRunListItemPrice = (run: IRun) => {
 
 export const getRunListItemInstance = (run: IRun) => {
     if (run.jobs.length > 1) {
-        return '';
+        return '-';
     }
 
     return run.latest_job_submission?.job_provisioning_data?.instance_type?.name;
@@ -65,7 +65,7 @@ export const getRunListItemInstance = (run: IRun) => {
 
 export const getRunListItemInstanceId = (run: IRun) => {
     if (run.jobs.length > 1) {
-        return '';
+        return '-';
     }
 
     return run.latest_job_submission?.job_provisioning_data?.instance_id ?? '-';
@@ -73,7 +73,7 @@ export const getRunListItemInstanceId = (run: IRun) => {
 
 export const getRunListItemRegion = (run: IRun) => {
     if (run.jobs.length > 1) {
-        return '';
+        return '-';
     }
 
     return run.latest_job_submission?.job_provisioning_data?.region ?? '-';
@@ -81,7 +81,7 @@ export const getRunListItemRegion = (run: IRun) => {
 
 export const getRunListItemBackend = (run: IRun) => {
     if (run.jobs.length > 1) {
-        return '';
+        return '-';
     }
 
     return run.latest_job_submission?.job_provisioning_data?.backend ?? '-';
@@ -92,3 +92,16 @@ export const getRunListItemServiceUrl = (run: IRun) => {
     if (!url) return null;
     return url.startsWith('/') ? `${getBaseUrl()}${url}` : url;
 };
+
+export const getRunListItemSchedule = (run: IRun) => {
+    if (run.run_spec.configuration.type != 'task' || !run.run_spec.configuration.schedule) return null;
+
+    return run.run_spec.configuration.schedule.cron.join(', ');
+};
+
+export const getRunListFinishedAt = (run: IRun) => {
+    if (!run.latest_job_submission || !run.latest_job_submission.finished_at || !finishedRunStatuses.includes(run.status)) {
+        return null;
+    }
+    return run.latest_job_submission.finished_at;
+};

{dstack-0.19.33 → dstack-0.19.34}/frontend/src/pages/Runs/List/hooks/useColumnsDefinitions.tsx

@@ -18,6 +18,7 @@ import { ROUTES } from 'routes';
 import { finishedRunStatuses } from 'pages/Runs/constants';
 
 import {
+    getRunListFinishedAt,
     getRunListItemBackend,
     getRunListItemInstance,
     getRunListItemPrice,
@@ -68,7 +69,10 @@ export const useColumnsDefinitions = () => {
         {
             id: 'finished_at',
             header: t('projects.run.finished_at'),
-            cell: (item: IRun) => (item.terminated_at ? format(new Date(item.terminated_at), DATE_TIME_FORMAT) : null),
+            cell: (item: IRun) => {
+                const finishedAt = getRunListFinishedAt(item);
+                return finishedAt ? format(new Date(finishedAt), DATE_TIME_FORMAT) : '-';
+            },
         },
         {
             id: 'status',

{dstack-0.19.33 → dstack-0.19.34}/frontend/src/types/run.d.ts

@@ -136,6 +136,7 @@ declare type TStopRunsRequestParams = {
 declare type TJobMetricsRequestParams = {
     project_name: IProject['project_name'];
     run_name: string;
+    run_id: string;
     replica_num?: number;
     job_num: number;
     limit?: number;
@@ -227,6 +228,7 @@ declare interface IJobSubmission {
     submission_num: number;
     status: TJobStatus;
     submitted_at: number;
+    finished_at: string | null;
     termination_reason?: string | null;
     termination_reason_message?: string | null;
     exit_status?: number | null;
@@ -239,9 +241,14 @@ declare interface IJob {
     job_submissions: IJobSubmission[];
 }
 
+declare interface ISchedule {
+    cron: string[];
+}
+
 declare interface ITaskConfiguration {
     type: 'task';
     priority?: number | null;
+    schedule?: ISchedule | null;
 }
 
 declare interface IServiceConfiguration {
@@ -249,6 +256,7 @@ declare interface IServiceConfiguration {
     gateway: string | null;
     priority?: number | null;
 }
+
 declare interface IRunSpec {
     configuration: TDevEnvironmentConfiguration | ITaskConfiguration | IServiceConfiguration;
     configuration_path: string;
@@ -276,7 +284,6 @@ declare interface IRun {
     project_name: string;
     user: string;
     submitted_at: string;
-    terminated_at: string | null;
     status: TJobStatus;
     error?: string | null;
     jobs: IJob[];
@@ -285,6 +292,7 @@ declare interface IRun {
     cost: number;
     service: IRunService | null;
     status_message?: string | null;
+    next_triggered_at?: string | null;
 }
 
 declare interface IMetricsItem {

{dstack-0.19.33 → dstack-0.19.34}/pyproject.toml

@@ -32,7 +32,7 @@ dependencies = [
     "python-multipart>=0.0.16",
     "filelock",
     "psutil",
-    "gpuhunt==0.1.10",
+    "gpuhunt==0.1.11",
     "argcomplete>=3.5.0",
     "ignore-python>=0.2.0",
     "orjson",

{dstack-0.19.33 → dstack-0.19.34}/src/dstack/_internal/core/backends/base/offers.py

@@ -23,7 +23,8 @@ SUPPORTED_GPUHUNT_FLAGS = [
     "oci-spot",
     "lambda-arm",
     "gcp-a4",
-    "gcp-g4-preview",
+    "gcp-g4",
+    "gcp-dws-calendar-mode",
 ]
 
 
@@ -94,6 +95,7 @@ def catalog_item_to_offer(
         ),
         region=item.location,
         price=item.price,
+        backend_data=item.provider_data,
     )
 
 

{dstack-0.19.33 → dstack-0.19.34}/src/dstack/_internal/core/backends/gcp/compute.py

@@ -90,6 +90,10 @@ RESOURCE_NAME_PATTERN = re.compile(r"[a-z0-9-]+")
 TPU_VERSIONS = [tpu.name for tpu in KNOWN_TPUS]
 
 
+class GCPOfferBackendData(CoreModel):
+    is_dws_calendar_mode: bool = False
+
+
 class GCPVolumeDiskBackendData(CoreModel):
     type: Literal["disk"] = "disk"
     disk_type: str
@@ -142,19 +146,13 @@ class GCPCompute(
         offer_keys_to_offers = {}
         offers_with_availability = []
         for offer in offers:
-            preview = False
-            if offer.instance.name.startswith("g4-standard-"):
-                if self.config.preview_features and "g4" in self.config.preview_features:
-                    preview = True
-                else:
-                    continue
             region = offer.region[:-2]  # strip zone
             key = (_unique_instance_name(offer.instance), region)
             if key in offer_keys_to_offers:
                 offer_keys_to_offers[key].availability_zones.append(offer.region)
                 continue
             availability = InstanceAvailability.NO_QUOTA
-            if preview or _has_gpu_quota(quotas[region], offer.instance.resources):
+            if _has_gpu_quota(quotas[region], offer.instance.resources):
                 availability = InstanceAvailability.UNKNOWN
             # todo quotas: cpu, memory, global gpu, tpu
             offer_with_availability = InstanceOfferWithAvailability(
@@ -202,6 +200,23 @@ class GCPCompute(
         modifiers.append(get_offers_disk_modifier(CONFIGURABLE_DISK_SIZE, requirements))
         return modifiers
 
+    def get_offers_post_filter(
+        self, requirements: Requirements
+    ) -> Optional[Callable[[InstanceOfferWithAvailability], bool]]:
+        if requirements.reservation is None:
+
+            def reserved_offers_filter(offer: InstanceOfferWithAvailability) -> bool:
+                """Remove reserved-only offers"""
+                if GCPOfferBackendData.__response__.parse_obj(
+                    offer.backend_data
+                ).is_dws_calendar_mode:
+                    return False
+                return True
+
+            return reserved_offers_filter
+
+        return None
+
     def terminate_instance(
         self, instance_id: str, region: str, backend_data: Optional[str] = None
     ) -> None:
@@ -1006,8 +1021,8 @@ def _has_gpu_quota(quotas: Dict[str, float], resources: Resources) -> bool:
     gpu = resources.gpus[0]
     if _is_tpu(gpu.name):
         return True
-    if gpu.name in ["B200", "H100"]:
-        # B200, H100 and H100_MEGA quotas are not returned by `regions_client.list`
+    if gpu.name in ["B200", "H100", "RTXPRO6000"]:
+        # B200, H100, H100_MEGA, and RTXPRO6000 quotas are not returned by `regions_client.list`
         return True
     quota_name = f"NVIDIA_{gpu.name}_GPUS"
     if gpu.name == "A100" and gpu.memory_mib == 80 * 1024:

{dstack-0.19.33 → dstack-0.19.34}/src/dstack/_internal/core/backends/gcp/models.py

@@ -92,7 +92,10 @@ class GCPBackendConfig(CoreModel):
     preview_features: Annotated[
         Optional[List[Literal["g4"]]],
         Field(
-            description=("The list of preview GCP features to enable. Supported values: `g4`"),
+            description=(
+                "The list of preview GCP features to enable."
+                " There are currently no preview features"
+            ),
             max_items=1,
         ),
     ] = None

{dstack-0.19.33 → dstack-0.19.34}/src/dstack/_internal/core/backends/runpod/compute.py

@@ -232,9 +232,12 @@ class RunpodCompute(
     def create_volume(self, volume: Volume) -> VolumeProvisioningData:
         volume_name = generate_unique_volume_name(volume, max_length=MAX_RESOURCE_NAME_LEN)
         size_gb = volume.configuration.size_gb
+        # Runpod regions must be uppercase.
+        # Lowercase regions are accepted in the API but they break Runpod in several ways.
+        region = volume.configuration.region.upper()
         volume_id = self.api_client.create_network_volume(
             name=volume_name,
-            region=volume.configuration.region,
+            region=region,
             size=size_gb,
         )
         return VolumeProvisioningData(

{dstack-0.19.33 → dstack-0.19.34}/src/dstack/_internal/core/compatibility/runs.py

@@ -33,6 +33,8 @@ def get_apply_plan_excludes(plan: ApplyRunPlanInput) -> Optional[IncludeExcludeD
             current_resource_excludes["deployment_num"] = True
         if current_resource.fleet is None:
             current_resource_excludes["fleet"] = True
+        if current_resource.next_triggered_at is None:
+            current_resource_excludes["next_triggered_at"] = True
         apply_plan_excludes["current_resource"] = current_resource_excludes
         current_resource_excludes["run_spec"] = get_run_spec_excludes(current_resource.run_spec)
         job_submissions_excludes: IncludeExcludeDictType = {}
@@ -47,10 +49,20 @@ def get_apply_plan_excludes(plan: ApplyRunPlanInput) -> Optional[IncludeExcludeD
             job_submissions_excludes["job_provisioning_data"] = {
                 "instance_type": {"resources": {"cpu_arch"}}
             }
+        jrd_offer_excludes = {}
+        if any(
+            js.job_runtime_data and js.job_runtime_data.offer for js in job_submissions
+        ) and all(
+            not js.job_runtime_data
+            or not js.job_runtime_data.offer
+            or not js.job_runtime_data.offer.backend_data
+            for js in job_submissions
+        ):
+            jrd_offer_excludes["backend_data"] = True
         if all(map(_should_exclude_job_submission_jrd_cpu_arch, job_submissions)):
-            job_submissions_excludes["job_runtime_data"] = {
-                "offer": {"instance": {"resources": {"cpu_arch"}}}
-            }
+            jrd_offer_excludes["instance"] = {"resources": {"cpu_arch"}}
+        if jrd_offer_excludes:
+            job_submissions_excludes["job_runtime_data"] = {"offer": jrd_offer_excludes}
         if all(js.exit_status is None for js in job_submissions):
             job_submissions_excludes["exit_status"] = True
         if all(js.status_message == "" for js in job_submissions):
@@ -69,9 +81,18 @@ def get_apply_plan_excludes(plan: ApplyRunPlanInput) -> Optional[IncludeExcludeD
                 latest_job_submission_excludes["job_provisioning_data"] = {
                     "instance_type": {"resources": {"cpu_arch"}}
                 }
+            latest_job_submission_jrd_offer_excludes = {}
+            if (
+                latest_job_submission.job_runtime_data
+                and latest_job_submission.job_runtime_data.offer
+                and not latest_job_submission.job_runtime_data.offer.backend_data
+            ):
+                latest_job_submission_jrd_offer_excludes["backend_data"] = True
             if _should_exclude_job_submission_jrd_cpu_arch(latest_job_submission):
+                latest_job_submission_jrd_offer_excludes["instance"] = {"resources": {"cpu_arch"}}
+            if latest_job_submission_jrd_offer_excludes:
                 latest_job_submission_excludes["job_runtime_data"] = {
-                    "offer": {"instance": {"resources": {"cpu_arch"}}}
+                    "offer": latest_job_submission_jrd_offer_excludes
                 }
             if latest_job_submission.exit_status is None:
                 latest_job_submission_excludes["exit_status"] = True

{dstack-0.19.33 → dstack-0.19.34}/src/dstack/_internal/core/models/instances.py

@@ -1,6 +1,6 @@
 import datetime
 from enum import Enum
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
 from uuid import UUID
 
 import gpuhunt
@@ -184,6 +184,7 @@ class InstanceOffer(CoreModel):
     instance: InstanceType
     region: str
     price: float
+    backend_data: dict[str, Any] = {}
 
 
 class InstanceOfferWithAvailability(InstanceOffer):

{dstack-0.19.33 → dstack-0.19.34}/src/dstack/_internal/core/models/runs.py

@@ -553,6 +553,7 @@ class Run(CoreModel):
     deployment_num: int = 0  # default for compatibility with pre-0.19.14 servers
     error: Optional[str] = None
     deleted: Optional[bool] = None
+    next_triggered_at: Optional[datetime] = None
 
     def is_deployment_in_progress(self) -> bool:
         return any(

dstack-0.19.34/src/dstack/_internal/core/services/ssh/key_manager.py

@@ -0,0 +1,56 @@
+import os
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import TYPE_CHECKING, Optional
+
+from dstack._internal.core.models.users import UserWithCreds
+
+if TYPE_CHECKING:
+    from dstack.api.server import APIClient
+
+KEY_REFRESH_RATE = timedelta(minutes=10)  # redownload the key periodically in case it was rotated
+
+
+@dataclass
+class UserSSHKey:
+    public_key: str
+    private_key_path: Path
+
+
+class UserSSHKeyManager:
+    def __init__(self, api_client: "APIClient", ssh_keys_dir: Path) -> None:
+        self._api_client = api_client
+        self._key_path = ssh_keys_dir / api_client.get_token_hash()
+        self._pub_key_path = self._key_path.with_suffix(".pub")
+
+    def get_user_key(self) -> Optional[UserSSHKey]:
+        """
+        Return the up-to-date user key, or None if the user has no key (if created before 0.19.33)
+        """
+        if (
+            not self._key_path.exists()
+            or not self._pub_key_path.exists()
+            or datetime.now() - datetime.fromtimestamp(self._key_path.stat().st_mtime)
+            > KEY_REFRESH_RATE
+        ):
+            if not self._download_user_key():
+                return None
+        return UserSSHKey(
+            public_key=self._pub_key_path.read_text(), private_key_path=self._key_path
+        )
+
+    def _download_user_key(self) -> bool:
+        user = self._api_client.users.get_my_user()
+        if not (isinstance(user, UserWithCreds) and user.ssh_public_key and user.ssh_private_key):
+            return False
+
+        def key_opener(path, flags):
+            return os.open(path, flags, 0o600)
+
+        with open(self._key_path, "w", opener=key_opener) as f:
+            f.write(user.ssh_private_key)
+        with open(self._pub_key_path, "w") as f:
+            f.write(user.ssh_public_key)
+
+        return True