dstack 0.19.19__py3-none-any.whl → 0.19.21__py3-none-any.whl

dstack/_internal/server/services/gateways/__init__.py

@@ -1,7 +1,7 @@
 import asyncio
 import datetime
 import uuid
-from datetime import timedelta, timezone
+from datetime import timedelta
 from functools import partial
 from typing import List, Optional, Sequence
 
@@ -11,16 +11,16 @@ from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import selectinload
 
 import dstack._internal.utils.random_names as random_names
-from dstack._internal.core.backends import (
-    BACKENDS_WITH_GATEWAY_SUPPORT,
-    BACKENDS_WITH_PRIVATE_GATEWAY_SUPPORT,
-)
 from dstack._internal.core.backends.base.compute import (
     Compute,
     ComputeWithGatewaySupport,
     get_dstack_gateway_wheel,
     get_dstack_runner_version,
 )
+from dstack._internal.core.backends.features import (
+    BACKENDS_WITH_GATEWAY_SUPPORT,
+    BACKENDS_WITH_PRIVATE_GATEWAY_SUPPORT,
+)
 from dstack._internal.core.errors import (
     GatewayError,
     ResourceNotExistsError,
@@ -86,15 +86,6 @@ async def get_gateway_by_name(
     return gateway_model_to_gateway(gateway)
 
 
-async def get_project_default_gateway(
-    session: AsyncSession, project: ProjectModel
-) -> Optional[Gateway]:
-    gateway: Optional[GatewayModel] = project.default_gateway
-    if gateway is None:
-        return None
-    return gateway_model_to_gateway(gateway)
-
-
 async def create_gateway_compute(
     project_name: str,
     backend_compute: Compute,
@@ -181,9 +172,9 @@ async def create_gateway(
         session.add(gateway)
         await session.commit()
 
-        if project.default_gateway is None or configuration.default:
+        default_gateway = await get_project_default_gateway_model(session=session, project=project)
+        if default_gateway is None or configuration.default:
             await set_default_gateway(session=session, project=project, name=configuration.name)
-
         return gateway_model_to_gateway(gateway)
 
 
@@ -349,6 +340,15 @@ async def get_project_gateway_model_by_name(
     return res.scalar()
 
 
+async def get_project_default_gateway_model(
+    session: AsyncSession, project: ProjectModel
+) -> Optional[GatewayModel]:
+    res = await session.execute(
+        select(GatewayModel).where(GatewayModel.id == project.default_gateway_id)
+    )
+    return res.scalar_one_or_none()
+
+
 async def generate_gateway_name(session: AsyncSession, project: ProjectModel) -> str:
     gateways = await list_project_gateway_models(session=session, project=project)
     names = {g.name for g in gateways}
@@ -557,7 +557,7 @@ def gateway_model_to_gateway(gateway_model: GatewayModel) -> Gateway:
         region=gateway_model.region,
         wildcard_domain=gateway_model.wildcard_domain,
         default=gateway_model.project.default_gateway_id == gateway_model.id,
-        created_at=gateway_model.created_at.replace(tzinfo=timezone.utc),
+        created_at=gateway_model.created_at,
         status=gateway_model.status,
         status_message=gateway_model.status_message,
         configuration=configuration,

dstack/_internal/server/services/instances.py

@@ -1,6 +1,6 @@
 import uuid
 from collections.abc import Container, Iterable
-from datetime import datetime, timezone
+from datetime import datetime
 from typing import Dict, List, Literal, Optional, Union
 
 import gpuhunt
@@ -8,11 +8,11 @@ from sqlalchemy import and_, or_, select
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import joinedload
 
-from dstack._internal.core.backends import BACKENDS_WITH_MULTINODE_SUPPORT
 from dstack._internal.core.backends.base.offers import (
     offer_to_catalog_item,
     requirements_to_query_filter,
 )
+from dstack._internal.core.backends.features import BACKENDS_WITH_MULTINODE_SUPPORT
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.envs import Env
 from dstack._internal.core.models.instances import (
@@ -34,7 +34,6 @@ from dstack._internal.core.models.profiles import (
     TerminationPolicy,
 )
 from dstack._internal.core.models.runs import JobProvisioningData, Requirements
-from dstack._internal.core.models.users import GlobalRole
 from dstack._internal.core.models.volumes import Volume
 from dstack._internal.core.services.profiles import get_termination
 from dstack._internal.server.models import (
@@ -44,7 +43,7 @@ from dstack._internal.server.models import (
     UserModel,
 )
 from dstack._internal.server.services.offers import generate_shared_offer
-from dstack._internal.server.services.projects import list_project_models, list_user_project_models
+from dstack._internal.server.services.projects import list_user_project_models
 from dstack._internal.utils import common as common_utils
 from dstack._internal.utils.logging import get_logger
 
@@ -62,7 +61,7 @@ def instance_model_to_instance(instance_model: InstanceModel) -> Instance:
         status=instance_model.status,
         unreachable=instance_model.unreachable,
         termination_reason=instance_model.termination_reason,
-        created=instance_model.created_at.replace(tzinfo=timezone.utc),
+        created=instance_model.created_at,
         total_blocks=instance_model.total_blocks,
         busy_blocks=instance_model.busy_blocks,
     )
@@ -372,18 +371,15 @@ async def list_user_instances(
     limit: int,
     ascending: bool,
 ) -> List[Instance]:
-    if user.global_role == GlobalRole.ADMIN:
-        projects = await list_project_models(session=session)
-    else:
-        projects = await list_user_project_models(session=session, user=user)
-    if not projects:
-        return []
-
+    projects = await list_user_project_models(
+        session=session,
+        user=user,
+        only_names=True,
+    )
     if project_names is not None:
-        projects = [proj for proj in projects if proj.name in project_names]
+        projects = [p for p in projects if p.name in project_names]
         if len(projects) == 0:
             return []
-
     instance_models = await list_projects_instance_models(
         session=session,
         projects=projects,

dstack/_internal/server/services/jobs/__init__.py

@@ -1,13 +1,13 @@
 import itertools
 import json
-from datetime import timedelta, timezone
+from datetime import timedelta
 from typing import Dict, Iterable, List, Optional, Tuple
 from uuid import UUID
 
 import requests
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
-from sqlalchemy.orm import joinedload
+from sqlalchemy.orm import joinedload, load_only
 
 import dstack._internal.server.services.backends as backends_services
 from dstack._internal.core.backends.base.backend import Backend
@@ -130,7 +130,7 @@ def job_model_to_job_submission(job_model: JobModel) -> JobSubmission:
         ):
             backend_data = json.loads(job_provisioning_data.backend_data)
             job_provisioning_data.backend = backend_data["base_backend"]
-    last_processed_at = job_model.last_processed_at.replace(tzinfo=timezone.utc)
+    last_processed_at = job_model.last_processed_at
     finished_at = None
     if job_model.status.is_finished():
         finished_at = last_processed_at
@@ -140,7 +140,7 @@ def job_model_to_job_submission(job_model: JobModel) -> JobSubmission:
         id=job_model.id,
         submission_num=job_model.submission_num,
         deployment_num=job_model.deployment_num,
-        submitted_at=job_model.submitted_at.replace(tzinfo=timezone.utc),
+        submitted_at=job_model.submitted_at,
         last_processed_at=last_processed_at,
         finished_at=finished_at,
         inactivity_secs=job_model.inactivity_secs,
@@ -231,10 +231,7 @@ async def process_terminating_job(
     Graceful stop should already be done by `process_terminating_run`.
     Caller must acquire the locks on the job and the job's instance.
     """
-    if (
-        job_model.remove_at is not None
-        and job_model.remove_at.replace(tzinfo=timezone.utc) > common.get_current_datetime()
-    ):
+    if job_model.remove_at is not None and job_model.remove_at > common.get_current_datetime():
         # it's too early to terminate the instance
         return
 
@@ -550,24 +547,25 @@ def _should_force_detach_volume(job_model: JobModel, stop_duration: Optional[int
     return (
         job_model.volumes_detached_at is not None
         and common.get_current_datetime()
-        > job_model.volumes_detached_at.replace(tzinfo=timezone.utc) + MIN_FORCE_DETACH_WAIT_PERIOD
+        > job_model.volumes_detached_at + MIN_FORCE_DETACH_WAIT_PERIOD
         and (
             job_model.termination_reason == JobTerminationReason.ABORTED_BY_USER
             or stop_duration is not None
             and common.get_current_datetime()
-            > job_model.volumes_detached_at.replace(tzinfo=timezone.utc)
-            + timedelta(seconds=stop_duration)
+            > job_model.volumes_detached_at + timedelta(seconds=stop_duration)
         )
     )
 
 
 async def get_instances_ids_with_detaching_volumes(session: AsyncSession) -> List[UUID]:
     res = await session.execute(
-        select(JobModel).where(
+        select(JobModel)
+        .where(
             JobModel.status == JobStatus.TERMINATING,
             JobModel.used_instance_id.is_not(None),
             JobModel.volumes_detached_at.is_not(None),
         )
+        .options(load_only(JobModel.used_instance_id))
     )
     job_models = res.scalars().all()
     return [jm.used_instance_id for jm in job_models if jm.used_instance_id]

dstack/_internal/server/services/logs/aws.py

@@ -55,6 +55,8 @@ class CloudWatchLogStorage(LogStorage):
     PAST_EVENT_MAX_DELTA = int((timedelta(days=14)).total_seconds()) * 1000 - CLOCK_DRIFT
     # "None of the log events in the batch can be more than 2 hours in the future."
     FUTURE_EVENT_MAX_DELTA = int((timedelta(hours=2)).total_seconds()) * 1000 - CLOCK_DRIFT
+    # Maximum number of retries when polling for log events to skip empty pages.
+    MAX_RETRIES = 10
 
     def __init__(self, *, group: str, region: Optional[str] = None) -> None:
         with self._wrap_boto_errors():
@@ -80,7 +82,7 @@ class CloudWatchLogStorage(LogStorage):
         next_token: Optional[str] = None
         with self._wrap_boto_errors():
             try:
-                cw_events, next_token = self._get_log_events(stream, request)
+                cw_events, next_token = self._get_log_events_with_retry(stream, request)
             except botocore.exceptions.ClientError as e:
                 if not self._is_resource_not_found_exception(e):
                     raise
@@ -101,7 +103,47 @@ class CloudWatchLogStorage(LogStorage):
             )
             for cw_event in cw_events
         ]
-        return JobSubmissionLogs(logs=logs, next_token=next_token if len(logs) > 0 else None)
+        return JobSubmissionLogs(logs=logs, next_token=next_token)
+
+    def _get_log_events_with_retry(
+        self, stream: str, request: PollLogsRequest
+    ) -> Tuple[List[_CloudWatchLogEvent], Optional[str]]:
+        current_request = request
+        previous_next_token = request.next_token
+
+        for attempt in range(self.MAX_RETRIES):
+            cw_events, next_token = self._get_log_events(stream, current_request)
+
+            if cw_events:
+                return cw_events, next_token
+
+            if not next_token or next_token == previous_next_token:
+                return [], None
+
+            previous_next_token = next_token
+            current_request = PollLogsRequest(
+                run_name=request.run_name,
+                job_submission_id=request.job_submission_id,
+                start_time=request.start_time,
+                end_time=request.end_time,
+                descending=request.descending,
+                next_token=next_token,
+                limit=request.limit,
+                diagnose=request.diagnose,
+            )
+
+        if not request.descending:
+            logger.debug(
+                "Stream %s: exhausted %d retries without finding logs, returning empty response",
+                stream,
+                self.MAX_RETRIES,
+            )
+        # Only return the next token after exhausting retries if going descending—
+        # AWS CloudWatch guarantees more logs in that case. In ascending mode,
+        # next token is always returned, even if no logs remain.
+        # So descending works reliably; ascending has limits if gaps are too large.
+        # In the future, UI/CLI should handle retries, and we can return next token for ascending too.
+        return [], next_token if request.descending else None
 
     def _get_log_events(
         self, stream: str, request: PollLogsRequest
@@ -115,7 +157,7 @@ class CloudWatchLogStorage(LogStorage):
         }
 
         if request.start_time:
-            parameters["startTime"] = datetime_to_unix_time_ms(request.start_time) + 1
+            parameters["startTime"] = datetime_to_unix_time_ms(request.start_time)
 
         if request.end_time:
             parameters["endTime"] = datetime_to_unix_time_ms(request.end_time)

dstack/_internal/server/services/logs/filelog.py

@@ -1,5 +1,6 @@
+import os
 from pathlib import Path
-from typing import List, Union
+from typing import Generator, List, Optional, Tuple, Union
 from uuid import UUID
 
 from dstack._internal.core.errors import ServerClientError
@@ -37,18 +38,17 @@ class FileLogStorage(LogStorage):
             producer=log_producer,
         )
 
+        if request.descending:
+            return self._poll_logs_descending(log_file_path, request)
+        else:
+            return self._poll_logs_ascending(log_file_path, request)
+
+    def _poll_logs_ascending(
+        self, log_file_path: Path, request: PollLogsRequest
+    ) -> JobSubmissionLogs:
         start_line = 0
         if request.next_token:
-            try:
-                start_line = int(request.next_token)
-                if start_line < 0:
-                    raise ServerClientError(
-                        f"Invalid next_token: {request.next_token}. Must be a non-negative integer."
-                    )
-            except ValueError:
-                raise ServerClientError(
-                    f"Invalid next_token: {request.next_token}. Must be a valid integer."
-                )
+            start_line = self._next_token(request)
 
         logs = []
         next_token = None
@@ -94,6 +94,102 @@ class FileLogStorage(LogStorage):
 
         return JobSubmissionLogs(logs=logs, next_token=next_token)
 
+    def _poll_logs_descending(
+        self, log_file_path: Path, request: PollLogsRequest
+    ) -> JobSubmissionLogs:
+        start_offset = self._next_token(request)
+
+        candidate_logs = []
+
+        try:
+            line_generator = self._read_lines_reversed(log_file_path, start_offset)
+
+            for line_bytes, line_start_offset in line_generator:
+                try:
+                    line_str = line_bytes.decode("utf-8")
+                    log_event = LogEvent.__response__.parse_raw(line_str)
+                except Exception:
+                    continue  # Skip malformed lines
+
+                if request.end_time is not None and log_event.timestamp > request.end_time:
+                    continue
+                if request.start_time and log_event.timestamp <= request.start_time:
+                    break
+
+                candidate_logs.append((log_event, line_start_offset))
+
+                if len(candidate_logs) > request.limit:
+                    break
+        except FileNotFoundError:
+            return JobSubmissionLogs(logs=[], next_token=None)
+
+        logs = [log for log, offset in candidate_logs[: request.limit]]
+        next_token = None
+        if len(candidate_logs) > request.limit:
+            # We fetched one more than the limit, so there are more pages.
+            # The next token should point to the start of the last log we are returning.
+            _last_log_event, last_log_offset = candidate_logs[request.limit - 1]
+            next_token = str(last_log_offset)
+
+        return JobSubmissionLogs(logs=logs, next_token=next_token)
+
+    @staticmethod
+    def _read_lines_reversed(
+        filepath: Path, start_offset: Optional[int] = None, chunk_size: int = 8192
+    ) -> Generator[Tuple[bytes, int], None, None]:
+        """
+        A generator that yields lines from a file in reverse order, along with the byte
+        offset of the start of each line. This is memory-efficient for large files.
+        """
+        with open(filepath, "rb") as f:
+            f.seek(0, os.SEEK_END)
+            file_size = f.tell()
+            cursor = file_size
+
+            # If a start_offset is provided, optimize by starting the read
+            # from a more specific location instead of the end of the file.
+            if start_offset is not None and start_offset < file_size:
+                # To get the full content of the line that straddles the offset,
+                # we need to find its end (the next newline character).
+                f.seek(start_offset)
+                chunk = f.read(chunk_size)
+                newline_pos = chunk.find(b"\n")
+                if newline_pos != -1:
+                    # Found the end of the line. The cursor for reverse reading
+                    # should start from this point to include the full line.
+                    cursor = start_offset + newline_pos + 1
+                else:
+                    # No newline found, which means the rest of the file is one line.
+                    # The default cursor pointing to file_size is correct.
+                    pass
+
+            buffer = b""
+
+            while cursor > 0:
+                seek_pos = max(0, cursor - chunk_size)
+                amount_to_read = cursor - seek_pos
+                f.seek(seek_pos)
+                chunk = f.read(amount_to_read)
+                cursor = seek_pos
+
+                buffer = chunk + buffer
+
+                while b"\n" in buffer:
+                    newline_pos = buffer.rfind(b"\n")
+                    line = buffer[newline_pos + 1 :]
+                    line_start_offset = cursor + newline_pos + 1
+
+                    # Skip lines that start at or after the start_offset
+                    if start_offset is None or line_start_offset < start_offset:
+                        yield line, line_start_offset
+
+                    buffer = buffer[:newline_pos]
+
+            # The remaining buffer is the first line of the file.
+            # Only yield it if we're not using start_offset or if it starts before start_offset
+            if buffer and (start_offset is None or 0 < start_offset):
+                yield buffer, 0
+
     def write_logs(
         self,
         project: ProjectModel,
@@ -148,3 +244,17 @@ class FileLogStorage(LogStorage):
             log_source=LogEventSource.STDOUT,
             message=runner_log_event.message.decode(errors="replace"),
         )
+
+    def _next_token(self, request: PollLogsRequest) -> Optional[int]:
+        next_token = request.next_token
+        if next_token is None:
+            return None
+        try:
+            value = int(next_token)
+            if value < 0:
+                raise ValueError("Offset must be non-negative")
+            return value
+        except (ValueError, TypeError):
+            raise ServerClientError(
+                f"Invalid next_token: {next_token}. Must be a non-negative integer."
+            )

dstack/_internal/server/services/offers.py

@@ -2,13 +2,13 @@ from typing import List, Literal, Optional, Tuple, Union
 
 import gpuhunt
 
-from dstack._internal.core.backends import (
+from dstack._internal.core.backends.base.backend import Backend
+from dstack._internal.core.backends.base.compute import ComputeWithPlacementGroupSupport
+from dstack._internal.core.backends.features import (
     BACKENDS_WITH_CREATE_INSTANCE_SUPPORT,
     BACKENDS_WITH_MULTINODE_SUPPORT,
     BACKENDS_WITH_RESERVATION_SUPPORT,
 )
-from dstack._internal.core.backends.base.backend import Backend
-from dstack._internal.core.backends.base.compute import ComputeWithPlacementGroupSupport
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.instances import (
     InstanceOfferWithAvailability,

dstack/_internal/server/services/projects.py

@@ -1,11 +1,10 @@
 import uuid
-from datetime import timezone
 from typing import Awaitable, Callable, List, Optional, Tuple
 
 from sqlalchemy import delete, select, update
 from sqlalchemy import func as safunc
 from sqlalchemy.ext.asyncio import AsyncSession
-from sqlalchemy.orm import joinedload
+from sqlalchemy.orm import QueryableAttribute, joinedload, load_only
 
 from dstack._internal.core.backends.configurators import get_configurator
 from dstack._internal.core.backends.dstack.models import (
@@ -54,13 +53,12 @@ async def list_user_projects(
     user: UserModel,
 ) -> List[Project]:
     """
-    Returns projects where the user is a member.
+    Returns projects where the user is a member or all projects for global admins.
     """
-    if user.global_role == GlobalRole.ADMIN:
-        projects = await list_project_models(session=session)
-    else:
-        projects = await list_user_project_models(session=session, user=user)
-
+    projects = await list_user_project_models(
+        session=session,
+        user=user,
+    )
     projects = sorted(projects, key=lambda p: p.created_at)
     return [
         project_model_to_project(p, include_backends=False, include_members=False)
@@ -80,7 +78,7 @@ async def list_user_accessible_projects(
     if user.global_role == GlobalRole.ADMIN:
         projects = await list_project_models(session=session)
     else:
-        member_projects = await list_user_project_models(session=session, user=user)
+        member_projects = await list_member_project_models(session=session, user=user)
         public_projects = await list_public_non_member_project_models(session=session, user=user)
         projects = member_projects + public_projects
 
@@ -167,7 +165,7 @@ async def delete_projects(
     projects_names: List[str],
 ):
     if user.global_role != GlobalRole.ADMIN:
-        user_projects = await list_user_project_models(
+        user_projects = await list_member_project_models(
             session=session, user=user, include_members=True
         )
         user_project_names = [p.name for p in user_projects]
@@ -339,9 +337,25 @@ async def clear_project_members(
 
 
 async def list_user_project_models(
+    session: AsyncSession,
+    user: UserModel,
+    only_names: bool = False,
+) -> List[ProjectModel]:
+    load_only_attrs = []
+    if only_names:
+        load_only_attrs += [ProjectModel.id, ProjectModel.name]
+    if user.global_role == GlobalRole.ADMIN:
+        return await list_project_models(session=session, load_only_attrs=load_only_attrs)
+    return await list_member_project_models(
+        session=session, user=user, load_only_attrs=load_only_attrs
+    )
+
+
+async def list_member_project_models(
     session: AsyncSession,
     user: UserModel,
     include_members: bool = False,
+    load_only_attrs: Optional[List[QueryableAttribute]] = None,
 ) -> List[ProjectModel]:
     """
     List project models for a user where they are a member.
@@ -349,6 +363,8 @@ async def list_user_project_models(
     options = []
     if include_members:
         options.append(joinedload(ProjectModel.members))
+    if load_only_attrs:
+        options.append(load_only(*load_only_attrs))
     res = await session.execute(
         select(ProjectModel)
         .where(
@@ -395,13 +411,20 @@ async def list_user_owned_project_models(
 
 async def list_project_models(
     session: AsyncSession,
+    load_only_attrs: Optional[List[QueryableAttribute]] = None,
 ) -> List[ProjectModel]:
+    options = []
+    if load_only_attrs:
+        options.append(load_only(*load_only_attrs))
     res = await session.execute(
-        select(ProjectModel).where(ProjectModel.deleted == False),
+        select(ProjectModel).where(ProjectModel.deleted == False).options(*options)
     )
     return list(res.scalars().all())
 
 
+# TODO: Do not load ProjectModel.backends and ProjectModel.members by default when getting project
+
+
 async def get_project_model_by_name(
     session: AsyncSession, project_name: str, ignore_case: bool = True
 ) -> Optional[ProjectModel]:
@@ -415,7 +438,6 @@ async def get_project_model_by_name(
         .where(*filters)
         .options(joinedload(ProjectModel.backends))
         .options(joinedload(ProjectModel.members))
-        .options(joinedload(ProjectModel.default_gateway))
     )
     return res.unique().scalar()
 
@@ -432,7 +454,6 @@ async def get_project_model_by_name_or_error(
         )
         .options(joinedload(ProjectModel.backends))
         .options(joinedload(ProjectModel.members))
-        .options(joinedload(ProjectModel.default_gateway))
     )
     return res.unique().scalar_one()
 
@@ -449,7 +470,6 @@ async def get_project_model_by_id_or_error(
         )
         .options(joinedload(ProjectModel.backends))
         .options(joinedload(ProjectModel.members))
-        .options(joinedload(ProjectModel.default_gateway))
     )
     return res.unique().scalar_one()
 
@@ -537,7 +557,7 @@ def project_model_to_project(
         project_id=project_model.id,
         project_name=project_model.name,
         owner=users.user_model_to_user(project_model.owner),
-        created_at=project_model.created_at.replace(tzinfo=timezone.utc),
+        created_at=project_model.created_at,
         backends=backends,
         members=members,
         is_public=project_model.is_public,

dstack/_internal/server/services/prometheus/client_metrics.py

@@ -5,6 +5,9 @@ class RunMetrics:
     """Wrapper class for run-related Prometheus metrics."""
 
     def __init__(self):
+        # submit_to_provision_duration reflects real provisioning time
+        # but does not reflect how quickly provisioning processing works
+        # since it includes scheduling time, retrying, etc.
         self._submit_to_provision_duration = Histogram(
             "dstack_submit_to_provision_duration_seconds",
             "Time from when a run has been submitted and first job provisioning",