dstack 0.19.19__tar.gz → 0.19.20__tar.gz

{dstack-0.19.19 → dstack-0.19.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dstack
-Version: 0.19.19
+Version: 0.19.20
 Summary: dstack is an open-source orchestration engine for running AI workloads on any cloud or on-premises.
 Project-URL: Homepage, https://dstack.ai
 Project-URL: Source, https://github.com/dstackai/dstack

{dstack-0.19.19 → dstack-0.19.20}/docs/docs/guides/metrics.md

@@ -43,9 +43,9 @@ To enable exporting metrics to Prometheus, set the
 In addition to the essential metrics available via the CLI and UI, `dstack` exports additional metrics to Prometheus, including data on fleets, runs, jobs, and DCGM metrics.
 
 ??? info "NVIDIA DCGM"
-    NVIDIA DCGM metrics are automatically collected for `aws`, `azure`, `gcp`, and `oci` backends, 
+    NVIDIA DCGM metrics are automatically collected for `aws`, `azure`, `gcp`, and `oci` backends,
     as well as for [SSH fleets](../concepts/fleets.md#ssh).
-    
+
     To ensure NVIDIA DCGM metrics are collected from SSH fleets, ensure the `datacenter-gpu-manager-4-core`,
     `datacenter-gpu-manager-4-proprietary`, and `datacenter-gpu-manager-exporter` packages are installed on the hosts.
 
@@ -112,6 +112,9 @@ telemetry, and more.
     | `dstack_job_memory_total_bytes`                 | *gauge*   | Total memory allocated for the job, bytes                                                  | `4009754624.0` |
     | `dstack_job_memory_usage_bytes`                 | *gauge*   | Memory used by the job (including cache), bytes                                            | `339017728.0`  |
     | `dstack_job_memory_working_set_bytes`           | *gauge*   | Memory used by the job (not including cache), bytes                                        | `147251200.0`  |
+    | `dstack_job_gpu_usage_ratio`                    | *gauge*   | Job GPU usage, percent (as 0.0-1.0)                                                        | `0.93`         |
+    | `dstack_job_gpu_memory_total_bytes`             | *gauge*   | Total GPU memory allocated for the job, bytes                                              | `8589934592.0` |
+    | `dstack_job_gpu_memory_usage_bytes`             | *gauge*   | GPU memory used by the job, bytes                                                          | `1048576.0`    |
     | `DCGM_FI_DEV_GPU_UTIL`                          | *gauge*   | GPU utilization (in %)                                                                     |                |
     | `DCGM_FI_DEV_MEM_COPY_UTIL`                     | *gauge*   | Memory utilization (in %)                                                                  |                |
     | `DCGM_FI_DEV_ENC_UTIL`                          | *gauge*   | Encoder utilization (in %)                                                                 |                |
@@ -176,6 +179,9 @@ telemetry, and more.
     | `dstack_run_type`     | *string*  | Run configuration type | `task`, `dev-environment`              |
     | `dstack_backend`      | *string*  | Backend                | `aws`, `runpod`                        |
     | `dstack_gpu`          | *string?* | GPU name               | `H100`                                 |
+    | `dstack_gpu_num`[^1]  | *integer* | GPU number (0-based)   | `0`                                    |
+
+    [^1]: For `dstack_gpu_*` metrics only.
 
 ### Server health metrics
 

{dstack-0.19.19 → dstack-0.19.20}/frontend/package-lock.json

@@ -16,8 +16,6 @@
         "@cloudscape-design/global-styles": "^1.0.33",
         "@hookform/resolvers": "^2.9.10",
         "@reduxjs/toolkit": "^1.9.1",
-        "@xterm/addon-fit": "^0.10.0",
-        "@xterm/xterm": "^5.5.0",
         "ace-builds": "^1.36.3",
         "classnames": "^2.5.1",
         "css-minimizer-webpack-plugin": "^4.2.2",
@@ -5665,19 +5663,6 @@
         }
       }
     },
-    "node_modules/@xterm/addon-fit": {
-      "version": "0.10.0",
-      "resolved": "https://registry.npmjs.org/@xterm/addon-fit/-/addon-fit-0.10.0.tgz",
-      "integrity": "sha512-UFYkDm4HUahf2lnEyHvio51TNGiLK66mqP2JoATy7hRZeXaGMRDr00JiSF7m63vR5WKATF605yEggJKsw0JpMQ==",
-      "peerDependencies": {
-        "@xterm/xterm": "^5.0.0"
-      }
-    },
-    "node_modules/@xterm/xterm": {
-      "version": "5.5.0",
-      "resolved": "https://registry.npmjs.org/@xterm/xterm/-/xterm-5.5.0.tgz",
-      "integrity": "sha512-hqJHYaQb5OptNunnyAnkHyM8aCjZ1MEIDTQu1iIbbTD/xops91NB5yq1ZK/dC2JDbVWtF23zUtl9JE2NqwT87A=="
-    },
     "node_modules/@xtuc/ieee754": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz",

{dstack-0.19.19 → dstack-0.19.20}/frontend/package.json

@@ -105,8 +105,6 @@
     "@cloudscape-design/global-styles": "^1.0.33",
     "@hookform/resolvers": "^2.9.10",
     "@reduxjs/toolkit": "^1.9.1",
-    "@xterm/addon-fit": "^0.10.0",
-    "@xterm/xterm": "^5.5.0",
     "ace-builds": "^1.36.3",
     "classnames": "^2.5.1",
     "css-minimizer-webpack-plugin": "^4.2.2",

{dstack-0.19.19 → dstack-0.19.20}/frontend/src/components/Code/index.tsx

@@ -1,4 +1,4 @@
-import React from 'react';
+import React, { forwardRef } from 'react';
 import classNames from 'classnames';
 import Box from '@cloudscape-design/components/box';
 
@@ -8,12 +8,12 @@ export interface Props extends React.PropsWithChildren {
     className?: string;
 }
 
-export const Code: React.FC<Props> = ({ children, className }) => {
+export const Code = forwardRef<HTMLDivElement, Props>(({ children, className }, ref) => {
     return (
-        <div className={classNames(styles.code, className)}>
+        <div ref={ref} className={classNames(styles.code, className)}>
             <Box variant="code" color="text-status-inactive">
                 {children}
             </Box>
         </div>
     );
-};
+});

{dstack-0.19.19 → dstack-0.19.20}/frontend/src/components/index.ts

@@ -56,7 +56,6 @@ export { default as PropertyFilter } from '@cloudscape-design/components/propert
 export type { PropertyFilterProps } from '@cloudscape-design/components/property-filter';
 export type { LineChartProps } from '@cloudscape-design/components/line-chart/interfaces';
 export type { ModalProps } from '@cloudscape-design/components/modal';
-export type { TilesProps } from '@cloudscape-design/components/tiles';
 
 // custom components
 export { NavigateLink } from './NavigateLink';

{dstack-0.19.19 → dstack-0.19.20}/frontend/src/index.tsx

@@ -12,7 +12,6 @@ import 'ace-builds/css/ace.css';
 import 'ace-builds/css/theme/cloud_editor.css';
 import 'ace-builds/css/theme/cloud_editor_dark.css';
 import 'assets/css/index.css';
-import '@xterm/xterm/css/xterm.css';
 
 import 'locale';
 

dstack-0.19.20/frontend/src/pages/Runs/Details/Logs/helpers.ts

@@ -0,0 +1,24 @@
+export const getJobSubmissionId = (run?: IRun): string | undefined => {
+    if (!run) return;
+
+    const lastJob = run.jobs[run.jobs.length - 1];
+
+    if (!lastJob) return;
+
+    return lastJob.job_submissions[lastJob.job_submissions.length - 1]?.id;
+};
+
+export const decodeLogs = (logs: ILogItem[]): ILogItem[] => {
+    return logs.map((log: ILogItem) => {
+        let { message } = log;
+
+        try {
+            message = atob(message);
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        } catch (e) {
+            return log;
+        }
+
+        return { ...log, message };
+    });
+};

dstack-0.19.20/frontend/src/pages/Runs/Details/Logs/index.tsx

@@ -0,0 +1,208 @@
+import React, { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react';
+import { useTranslation } from 'react-i18next';
+import classNames from 'classnames';
+
+import { Box, Button, Code, Container, Header, ListEmptyMessage, Loader, TextContent } from 'components';
+
+import { useLocalStorageState } from 'hooks/useLocalStorageState';
+import { useLazyGetProjectLogsQuery } from 'services/project';
+
+import { decodeLogs } from './helpers';
+
+import { IProps } from './types';
+
+import styles from './styles.module.scss';
+
+const LIMIT_LOG_ROWS = 100;
+const LOADING_SCROLL_GAP = 300;
+
+export const Logs: React.FC<IProps> = ({ className, projectName, runName, jobSubmissionId }) => {
+    const { t } = useTranslation();
+    const codeRef = useRef<HTMLDivElement>(null);
+    const nextTokenRef = useRef<string | undefined>(undefined);
+    const scrollPositionByBottom = useRef<number>(0);
+
+    const [logsData, setLogsData] = useState<ILogItem[]>([]);
+    const [isLoading, setIsLoading] = useState(false);
+    const [getProjectLogs] = useLazyGetProjectLogsQuery();
+    const [isEnabledDecoding, setIsEnabledDecoding] = useLocalStorageState('enable-encode-logs', false);
+    // const [isShowTimestamp, setIsShowTimestamp] = useLocalStorageState('enable-showing-timestamp-logs', false);
+
+    const logsForView = useMemo(() => {
+        if (isEnabledDecoding) {
+            return decodeLogs(logsData);
+        }
+
+        return logsData;
+    }, [logsData, isEnabledDecoding]);
+
+    const saveScrollPositionByBottom = () => {
+        if (!codeRef.current) return;
+
+        const { clientHeight, scrollHeight, scrollTop } = codeRef.current;
+        scrollPositionByBottom.current = scrollHeight - clientHeight - scrollTop;
+    };
+
+    const restoreScrollPositionByBottom = () => {
+        if (!codeRef.current) return;
+
+        const { clientHeight, scrollHeight } = codeRef.current;
+        codeRef.current.scrollTo(0, scrollHeight - clientHeight - scrollPositionByBottom.current);
+    };
+
+    const checkNeedMoreLoadingData = () => {
+        if (!codeRef.current) return;
+
+        const { clientHeight, scrollHeight } = codeRef.current;
+
+        if (scrollHeight - clientHeight <= LOADING_SCROLL_GAP) {
+            getNextLogItems();
+        }
+    };
+
+    const getLogItems = (nextToken?: string) => {
+        setIsLoading(true);
+
+        if (!jobSubmissionId) {
+            return;
+        }
+
+        getProjectLogs({
+            project_name: projectName,
+            run_name: runName,
+            descending: true,
+            job_submission_id: jobSubmissionId,
+            next_token: nextToken,
+            limit: LIMIT_LOG_ROWS,
+        })
+            .unwrap()
+            .then((response) => {
+                saveScrollPositionByBottom();
+                const reversed = response.logs.toReversed();
+
+                if (nextToken) {
+                    setLogsData((old) => [...reversed, ...old]);
+                } else {
+                    setLogsData(reversed);
+                }
+
+                nextTokenRef.current = response.next_token;
+                setIsLoading(false);
+            })
+            .catch(() => setIsLoading(false));
+    };
+
+    const getNextLogItems = () => {
+        if (nextTokenRef.current) {
+            getLogItems(nextTokenRef.current);
+        }
+    };
+
+    const toggleDecodeLogs = () => {
+        saveScrollPositionByBottom();
+        setIsEnabledDecoding(!isEnabledDecoding);
+    };
+
+    useEffect(() => {
+        getLogItems();
+    }, []);
+
+    useLayoutEffect(() => {
+        if (logsForView.length && logsForView.length <= LIMIT_LOG_ROWS) {
+            scrollToBottom();
+        } else {
+            restoreScrollPositionByBottom();
+        }
+
+        if (logsForView.length) checkNeedMoreLoadingData();
+    }, [logsForView]);
+
+    const onScroll = useCallback<EventListener>(
+        (event) => {
+            const element = event.target as HTMLDivElement;
+
+            if (element.scrollTop <= LOADING_SCROLL_GAP && !isLoading) {
+                getNextLogItems();
+            }
+        },
+        [isLoading, logsForView],
+    );
+
+    useEffect(() => {
+        if (!codeRef.current) return;
+
+        codeRef.current.addEventListener('scroll', onScroll);
+
+        return () => {
+            if (codeRef.current) codeRef.current.removeEventListener('scroll', onScroll);
+        };
+    }, [codeRef.current, onScroll]);
+
+    const scrollToBottom = () => {
+        if (!codeRef.current) return;
+
+        const { clientHeight, scrollHeight } = codeRef.current;
+        codeRef.current.scrollTo(0, scrollHeight - clientHeight);
+    };
+
+    return (
+        <div className={classNames(styles.logs, className)}>
+            <Container
+                header={
+                    <div className={styles.headerContainer}>
+                        <div className={styles.headerTitle}>
+                            <Header variant="h2">{t('projects.run.log')}</Header>
+                        </div>
+
+                        <Loader
+                            show={isLoading && Boolean(logsForView.length)}
+                            padding={'n'}
+                            className={styles.loader}
+                            loadingText={''}
+                        />
+
+                        <div className={styles.switchers}>
+                            <Box>
+                                <Button
+                                    ariaLabel="Legacy mode"
+                                    formAction="none"
+                                    iconName="gen-ai"
+                                    variant={isEnabledDecoding ? 'primary' : 'icon'}
+                                    onClick={toggleDecodeLogs}
+                                />
+                            </Box>
+
+                            {/*<Box>*/}
+                            {/*    <Toggle onChange={({ detail }) => setIsShowTimestamp(detail.checked)} checked={isShowTimestamp}>*/}
+                            {/*        Show timestamp*/}
+                            {/*    </Toggle>*/}
+                            {/*</Box>*/}
+                        </div>
+                    </div>
+                }
+            >
+                <TextContent>
+                    {!isLoading && !logsForView.length && (
+                        <ListEmptyMessage
+                            title={t('projects.run.log_empty_message_title')}
+                            message={t('projects.run.log_empty_message_text')}
+                        />
+                    )}
+
+                    {!logsForView.length && <Loader show={isLoading} className={styles.mainLoader} />}
+
+                    {Boolean(logsForView.length) && (
+                        <Code className={styles.terminal} ref={codeRef}>
+                            {logsForView.map((log, i) => (
+                                <p key={i}>
+                                    {/*{isShowTimestamp && <span className={styles.timestamp}>{log.timestamp}</span>}*/}
+                                    {log.message}
+                                </p>
+                            ))}
+                        </Code>
+                    )}
+                </TextContent>
+            </Container>
+        </div>
+    );
+};

{dstack-0.19.19 → dstack-0.19.20}/frontend/src/pages/Runs/Details/Logs/styles.module.scss

@@ -4,16 +4,33 @@
     display: flex;
     gap: 10px;
     align-items: center;
+    padding-top: 4px;
+
+    .headerTitle {
+        flex-shrink: 0;
+        margin-top: -4px;
+    }
+
+    .switchers {
+        margin-left: auto;
+        display: flex;
+        gap: 24px;
+    }
 }
 
 .loader {
     position: relative;
-    top: -2px;
     height: 20px;
     background-color: rgba(awsui.$color-background-container-content, .8);
     color: #6e6e6e;
 }
 
+.mainLoader {
+    margin-top: auto;
+    margin-bottom: auto;
+    transform: translateY(-24px);
+}
+
 .logs {
     display: flex;
     flex-direction: column;
@@ -56,6 +73,23 @@
                 .terminal {
                     flex-grow: 1;
                     min-height: 0;
+                    height: 0;
+                    overflow-y: auto;
+                    background-color: awsui.$color-background-layout-main;
+
+                    code {
+                        color: awsui.$color-text-body-default !important;
+                    }
+
+                    p {
+                        padding: 0 !important;
+                        font-size: awsui.$font-size-body-s !important;
+                        line-height: awsui.$line-height-body-s !important;
+
+                        .timestamp {
+                            padding-right: 8px;
+                        }
+                    }
                 }
 
                 .scroll {

{dstack-0.19.19 → dstack-0.19.20}/src/dstack/_internal/core/backends/cloudrift/api_client.py

@@ -155,8 +155,20 @@ class RiftClient:
         logger.debug("Terminating instance with request data: %s", request_data)
         response_data = self._make_request("instances/terminate", request_data)
         if isinstance(response_data, dict):
+            logger.debug("Terminating instance with response: %s", response_data)
             info = response_data.get("terminated", [])
-            return len(info) > 0
+            is_terminated = len(info) > 0
+            if not is_terminated:
+                # check if the instance is already terminated
+                instance_info = self.get_instance_by_id(instance_id)
+                is_terminated = instance_info is None or instance_info.get("status") == "Inactive"
+                logger.debug(
+                    "Instance %s is already terminated: %s response: %s",
+                    instance_id,
+                    is_terminated,
+                    instance_info,
+                )
+            return is_terminated
 
         return False
 

{dstack-0.19.19 → dstack-0.19.20}/src/dstack/_internal/core/backends/oci/resources.py

@@ -26,7 +26,7 @@ from dstack import version
 from dstack._internal.core.backends.oci.region import OCIRegionClient
 from dstack._internal.core.errors import BackendError
 from dstack._internal.core.models.instances import InstanceOffer
-from dstack._internal.utils.common import split_chunks
+from dstack._internal.utils.common import batched
 from dstack._internal.utils.logging import get_logger
 
 logger = get_logger(__name__)
@@ -667,21 +667,21 @@ def add_security_group_rules(
     security_group_id: str, rules: Iterable[SecurityRule], client: oci.core.VirtualNetworkClient
 ) -> None:
     rules_details = map(SecurityRule.to_sdk_add_rule_details, rules)
-    for chunk in split_chunks(rules_details, ADD_SECURITY_RULES_MAX_CHUNK_SIZE):
+    for batch in batched(rules_details, ADD_SECURITY_RULES_MAX_CHUNK_SIZE):
         client.add_network_security_group_security_rules(
             security_group_id,
-            oci.core.models.AddNetworkSecurityGroupSecurityRulesDetails(security_rules=chunk),
+            oci.core.models.AddNetworkSecurityGroupSecurityRulesDetails(security_rules=batch),
         )
 
 
 def remove_security_group_rules(
     security_group_id: str, rule_ids: Iterable[str], client: oci.core.VirtualNetworkClient
 ) -> None:
-    for chunk in split_chunks(rule_ids, REMOVE_SECURITY_RULES_MAX_CHUNK_SIZE):
+    for batch in batched(rule_ids, REMOVE_SECURITY_RULES_MAX_CHUNK_SIZE):
         client.remove_network_security_group_security_rules(
             security_group_id,
             oci.core.models.RemoveNetworkSecurityGroupSecurityRulesDetails(
-                security_rule_ids=chunk
+                security_rule_ids=batch
             ),
         )
 

{dstack-0.19.19 → dstack-0.19.20}/src/dstack/_internal/server/schemas/logs.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 from typing import Optional
 
-from pydantic import UUID4, Field, validator
+from pydantic import UUID4, Field
 
 from dstack._internal.core.models.common import CoreModel
 
@@ -15,11 +15,3 @@ class PollLogsRequest(CoreModel):
     next_token: Optional[str] = None
     limit: int = Field(100, ge=0, le=1000)
     diagnose: bool = False
-
-    @validator("descending")
-    @classmethod
-    def validate_descending(cls, v):
-        # Descending is not supported until we migrate from base64-encoded logs to plain text logs.
-        if v is True:
-            raise ValueError("descending: true is not supported")
-        return v

{dstack-0.19.19 → dstack-0.19.20}/src/dstack/_internal/server/services/logs/aws.py

@@ -55,6 +55,8 @@ class CloudWatchLogStorage(LogStorage):
     PAST_EVENT_MAX_DELTA = int((timedelta(days=14)).total_seconds()) * 1000 - CLOCK_DRIFT
     # "None of the log events in the batch can be more than 2 hours in the future."
     FUTURE_EVENT_MAX_DELTA = int((timedelta(hours=2)).total_seconds()) * 1000 - CLOCK_DRIFT
+    # Maximum number of retries when polling for log events to skip empty pages.
+    MAX_RETRIES = 10
 
     def __init__(self, *, group: str, region: Optional[str] = None) -> None:
         with self._wrap_boto_errors():
@@ -80,7 +82,7 @@ class CloudWatchLogStorage(LogStorage):
         next_token: Optional[str] = None
         with self._wrap_boto_errors():
             try:
-                cw_events, next_token = self._get_log_events(stream, request)
+                cw_events, next_token = self._get_log_events_with_retry(stream, request)
             except botocore.exceptions.ClientError as e:
                 if not self._is_resource_not_found_exception(e):
                     raise
@@ -101,7 +103,47 @@ class CloudWatchLogStorage(LogStorage):
             )
             for cw_event in cw_events
         ]
-        return JobSubmissionLogs(logs=logs, next_token=next_token if len(logs) > 0 else None)
+        return JobSubmissionLogs(logs=logs, next_token=next_token)
+
+    def _get_log_events_with_retry(
+        self, stream: str, request: PollLogsRequest
+    ) -> Tuple[List[_CloudWatchLogEvent], Optional[str]]:
+        current_request = request
+        previous_next_token = request.next_token
+
+        for attempt in range(self.MAX_RETRIES):
+            cw_events, next_token = self._get_log_events(stream, current_request)
+
+            if cw_events:
+                return cw_events, next_token
+
+            if not next_token or next_token == previous_next_token:
+                return [], None
+
+            previous_next_token = next_token
+            current_request = PollLogsRequest(
+                run_name=request.run_name,
+                job_submission_id=request.job_submission_id,
+                start_time=request.start_time,
+                end_time=request.end_time,
+                descending=request.descending,
+                next_token=next_token,
+                limit=request.limit,
+                diagnose=request.diagnose,
+            )
+
+        if not request.descending:
+            logger.debug(
+                "Stream %s: exhausted %d retries without finding logs, returning empty response",
+                stream,
+                self.MAX_RETRIES,
+            )
+        # Only return the next token after exhausting retries if going descending—
+        # AWS CloudWatch guarantees more logs in that case. In ascending mode,
+        # next token is always returned, even if no logs remain.
+        # So descending works reliably; ascending has limits if gaps are too large.
+        # In the future, UI/CLI should handle retries, and we can return next token for ascending too.
+        return [], next_token if request.descending else None
 
     def _get_log_events(
         self, stream: str, request: PollLogsRequest
@@ -115,7 +157,7 @@ class CloudWatchLogStorage(LogStorage):
         }
 
         if request.start_time:
-            parameters["startTime"] = datetime_to_unix_time_ms(request.start_time) + 1
+            parameters["startTime"] = datetime_to_unix_time_ms(request.start_time)
 
         if request.end_time:
             parameters["endTime"] = datetime_to_unix_time_ms(request.end_time)