dstack 0.19.15rc1__py3-none-any.whl → 0.19.17__py3-none-any.whl

dstack/_internal/server/services/logs/aws.py

@@ -78,14 +78,22 @@ class CloudWatchLogStorage(LogStorage):
             project.name, request.run_name, request.job_submission_id, log_producer
         )
         cw_events: List[_CloudWatchLogEvent]
+        next_token: Optional[str] = None
         with self._wrap_boto_errors():
             try:
-                cw_events = self._get_log_events(stream, request)
+                cw_events, next_token = self._get_log_events(stream, request)
             except botocore.exceptions.ClientError as e:
                 if not self._is_resource_not_found_exception(e):
                     raise
-                logger.debug("Stream %s not found, returning dummy response", stream)
-                cw_events = []
+                # Check if the group exists to distinguish between group not found vs stream not found
+                try:
+                    self._check_group_exists(self._group)
+                    # Group exists, so the error must be due to missing stream
+                    logger.debug("Stream %s not found, returning dummy response", stream)
+                    cw_events = []
+                except LogStorageError:
+                    # Group doesn't exist, re-raise the LogStorageError
+                    raise
         logs = [
             LogEvent(
                 timestamp=unix_time_ms_to_datetime(cw_event["timestamp"]),
@@ -94,51 +102,43 @@ class CloudWatchLogStorage(LogStorage):
             )
             for cw_event in cw_events
         ]
-        return JobSubmissionLogs(logs=logs)
+        return JobSubmissionLogs(logs=logs, next_token=next_token if len(logs) > 0 else None)
 
-    def _get_log_events(self, stream: str, request: PollLogsRequest) -> List[_CloudWatchLogEvent]:
-        limit = request.limit
+    def _get_log_events(
+        self, stream: str, request: PollLogsRequest
+    ) -> Tuple[List[_CloudWatchLogEvent], Optional[str]]:
+        start_from_head = not request.descending
         parameters = {
             "logGroupName": self._group,
             "logStreamName": stream,
-            "limit": limit,
+            "limit": request.limit,
+            "startFromHead": start_from_head,
         }
-        start_from_head = not request.descending
-        parameters["startFromHead"] = start_from_head
+
         if request.start_time:
-            # XXX: Since callers use start_time/end_time for pagination, one millisecond is added
-            # to avoid an infinite loop because startTime boundary is inclusive.
             parameters["startTime"] = datetime_to_unix_time_ms(request.start_time) + 1
+
         if request.end_time:
-            # No need to substract one millisecond in this case, though, seems that endTime is
-            # exclusive, that is, time interval boundaries are [startTime, entTime)
             parameters["endTime"] = datetime_to_unix_time_ms(request.end_time)
-        # "Partially full or empty pages don't necessarily mean that pagination is finished.
-        # As long as the nextBackwardToken or nextForwardToken returned is NOT equal to the
-        # nextToken that you passed into the API call, there might be more log events available."
-        events: List[_CloudWatchLogEvent] = []
-        next_token: Optional[str] = None
+        elif start_from_head:
+            # When startFromHead=true and no endTime is provided, set endTime to "now"
+            # to prevent infinite pagination as new logs arrive faster than we can read them
+            parameters["endTime"] = datetime_to_unix_time_ms(datetime.now(timezone.utc))
+
+        if request.next_token:
+            parameters["nextToken"] = request.next_token
+
+        response = self._client.get_log_events(**parameters)
+
+        events = response.get("events", [])
         next_token_key = "nextForwardToken" if start_from_head else "nextBackwardToken"
-        # Limit max tries to avoid a possible infinite loop if the API is misbehaving
-        tries_left = 10
-        while tries_left:
-            if next_token is not None:
-                parameters["nextToken"] = next_token
-            response = self._client.get_log_events(**parameters)
-            if start_from_head:
-                events.extend(response["events"])
-            else:
-                # Regardless of the startFromHead value log events are arranged in
-                # chronological order, from earliest to latest.
-                events.extend(reversed(response["events"]))
-            if len(events) >= limit:
-                return events[:limit]
-            if response[next_token_key] == next_token:
-                return events
-            next_token = response[next_token_key]
-            tries_left -= 1
-        logger.warning("too many requests to stream %s, returning partial response", stream)
-        return events
+        next_token = response.get(next_token_key)
+
+        # TODO: The code below is not going to be used until we migrate from base64-encoded logs to plain text logs.
+        if request.descending:
+            events = list(reversed(events))
+
+        return events, next_token
 
     def write_logs(
         self,

dstack/_internal/server/services/logs/filelog.py

@@ -14,6 +14,7 @@ from dstack._internal.server.schemas.logs import PollLogsRequest
 from dstack._internal.server.schemas.runner import LogEvent as RunnerLogEvent
 from dstack._internal.server.services.logs.base import (
     LogStorage,
+    LogStorageError,
     b64encode_raw_message,
     unix_time_ms_to_datetime,
 )
@@ -29,7 +30,9 @@ class FileLogStorage(LogStorage):
             self.root = Path(root)
 
     def poll_logs(self, project: ProjectModel, request: PollLogsRequest) -> JobSubmissionLogs:
-        # TODO Respect request.limit to support pagination
+        if request.descending:
+            raise LogStorageError("descending: true is not supported")
+
         log_producer = LogProducer.RUNNER if request.diagnose else LogProducer.JOB
         log_file_path = self._get_log_file_path(
             project_name=project.name,
@@ -37,22 +40,53 @@ class FileLogStorage(LogStorage):
             job_submission_id=request.job_submission_id,
             producer=log_producer,
         )
+
+        start_line = 0
+        if request.next_token:
+            try:
+                start_line = int(request.next_token)
+                if start_line < 0:
+                    raise LogStorageError(
+                        f"Invalid next_token: {request.next_token}. Must be a non-negative integer."
+                    )
+            except ValueError:
+                raise LogStorageError(
+                    f"Invalid next_token: {request.next_token}. Must be a valid integer."
+                )
+
         logs = []
+        next_token = None
+        current_line = 0
+
         try:
             with open(log_file_path) as f:
-                for line in f:
-                    log_event = LogEvent.__response__.parse_raw(line)
-                    if request.start_time and log_event.timestamp <= request.start_time:
-                        continue
-                    if request.end_time is None or log_event.timestamp < request.end_time:
-                        logs.append(log_event)
-                    else:
-                        break
-        except IOError:
-            pass
-        if request.descending:
-            logs = list(reversed(logs))
-        return JobSubmissionLogs(logs=logs)
+                lines = f.readlines()
+
+            for i, line in enumerate(lines):
+                if current_line < start_line:
+                    current_line += 1
+                    continue
+
+                log_event = LogEvent.__response__.parse_raw(line)
+                current_line += 1
+
+                if request.start_time and log_event.timestamp <= request.start_time:
+                    continue
+                if request.end_time is not None and log_event.timestamp >= request.end_time:
+                    break
+
+                logs.append(log_event)
+
+                if len(logs) >= request.limit:
+                    # Only set next_token if there are more lines to read
+                    if current_line < len(lines):
+                        next_token = str(current_line)
+                    break
+
+        except IOError as e:
+            raise LogStorageError(f"Failed to read log file {log_file_path}: {e}")
+
+        return JobSubmissionLogs(logs=logs, next_token=next_token)
 
     def write_logs(
         self,

dstack/_internal/server/services/logs/gcp.py

@@ -1,5 +1,4 @@
-import time
-from typing import Iterable, List
+from typing import List
 from uuid import UUID
 
 from dstack._internal.core.errors import ServerClientError
@@ -25,7 +24,8 @@ GCP_LOGGING_AVAILABLE = True
 try:
     import google.api_core.exceptions
     import google.auth.exceptions
-    from google.cloud import logging
+    from google.cloud import logging_v2
+    from google.cloud.logging_v2.types import ListLogEntriesRequest
 except ImportError:
     GCP_LOGGING_AVAILABLE = False
 
@@ -50,7 +50,7 @@ class GCPLogStorage(LogStorage):
 
     def __init__(self, project_id: str):
         try:
-            self.client = logging.Client(project=project_id)
+            self.client = logging_v2.Client(project=project_id)
             self.logger = self.client.logger(name=self.LOG_NAME)
             self.logger.list_entries(max_results=1)
             # Python client doesn't seem to support dry_run,
@@ -64,6 +64,7 @@ class GCPLogStorage(LogStorage):
             raise LogStorageError("Insufficient permissions")
 
     def poll_logs(self, project: ProjectModel, request: PollLogsRequest) -> JobSubmissionLogs:
+        # TODO: GCP may return logs in random order when events have the same timestamp.
         producer = LogProducer.RUNNER if request.diagnose else LogProducer.JOB
         stream_name = self._get_stream_name(
             project_name=project.name,
@@ -78,23 +79,27 @@ class GCPLogStorage(LogStorage):
             log_filters.append(f'timestamp < "{request.end_time.isoformat()}"')
         log_filter = " AND ".join(log_filters)
 
-        order_by = logging.DESCENDING if request.descending else logging.ASCENDING
+        order_by = logging_v2.DESCENDING if request.descending else logging_v2.ASCENDING
         try:
-            entries: Iterable[logging.LogEntry] = self.logger.list_entries(
-                filter_=log_filter,
+            # Use low-level API to get access to next_page_token
+            request_obj = ListLogEntriesRequest(
+                resource_names=[f"projects/{self.client.project}"],
+                filter=log_filter,
                 order_by=order_by,
-                max_results=request.limit,
-                # Specify max possible page_size (<=1000) to reduce number of API calls.
                 page_size=request.limit,
+                page_token=request.next_token,
             )
+            response = self.client._logging_api._gapic_api.list_log_entries(request=request_obj)
+
             logs = [
                 LogEvent(
                     timestamp=entry.timestamp,
-                    message=entry.payload["message"],
+                    message=entry.json_payload.get("message"),
                     log_source=LogEventSource.STDOUT,
                 )
-                for entry in entries
+                for entry in response.entries
             ]
+            next_token = response.next_page_token or None
         except google.api_core.exceptions.ResourceExhausted as e:
             logger.warning("GCP Logging exception: %s", repr(e))
             # GCP Logging has severely low quota of 60 reads/min for entries.list
@@ -102,11 +107,7 @@ class GCPLogStorage(LogStorage):
                 "GCP Logging read request limit exceeded."
                 " It's recommended to increase default entries.list request quota from 60 per minute."
             )
-        # We intentionally make reading logs slow to prevent hitting GCP quota.
-        # This doesn't help with many concurrent clients but
-        # should help with one client reading all logs sequentially.
-        time.sleep(1)
-        return JobSubmissionLogs(logs=logs)
+        return JobSubmissionLogs(logs=logs, next_token=next_token if len(logs) > 0 else None)
 
     def write_logs(
         self,

dstack/_internal/server/services/projects.py

@@ -74,8 +74,8 @@ async def list_user_accessible_projects(
 ) -> List[Project]:
     """
     Returns all projects accessible to the user:
-    - For global admins: ALL projects in the system
-    - For regular users: Projects where user is a member + public projects where user is NOT a member
+    - Projects where user is a member (public or private)
+    - Public projects where user is NOT a member
     """
     if user.global_role == GlobalRole.ADMIN:
         projects = await list_project_models(session=session)
@@ -150,6 +150,17 @@ async def create_project(
     return project_model_to_project(project_model)
 
 
+async def update_project(
+    session: AsyncSession,
+    user: UserModel,
+    project: ProjectModel,
+    is_public: bool,
+):
+    """Update project visibility (public/private)."""
+    project.is_public = is_public
+    await session.commit()
+
+
 async def delete_projects(
     session: AsyncSession,
     user: UserModel,
@@ -163,7 +174,8 @@ async def delete_projects(
         for project_name in projects_names:
             if project_name not in user_project_names:
                 raise ForbiddenError()
-        for project in user_projects:
+        projects_to_delete = [p for p in user_projects if p.name in projects_names]
+        for project in projects_to_delete:
             if not _is_project_admin(user=user, project=project):
                 raise ForbiddenError()
         if all(name in projects_names for name in user_project_names):
@@ -187,7 +199,6 @@ async def set_project_members(
     project: ProjectModel,
     members: List[MemberSetting],
 ):
-    # reload with members
     project = await get_project_model_by_name_or_error(
         session=session,
         project_name=project.name,
@@ -212,7 +223,6 @@ async def set_project_members(
         select(UserModel).where((UserModel.name.in_(names)) | (UserModel.email.in_(names)))
     )
     users = res.scalars().all()
-    # Create lookup maps for both username and email
     username_to_user = {user.name: user for user in users}
     email_to_user = {user.email: user for user in users if user.email}
     for i, member in enumerate(members):
@@ -230,6 +240,77 @@ async def set_project_members(
     await session.commit()
 
 
+async def add_project_members(
+    session: AsyncSession,
+    user: UserModel,
+    project: ProjectModel,
+    members: List[MemberSetting],
+):
+    """Add multiple members to a project."""
+    project = await get_project_model_by_name_or_error(
+        session=session,
+        project_name=project.name,
+    )
+    requesting_user_role = get_user_project_role(user=user, project=project)
+
+    is_self_join_to_public = (
+        len(members) == 1
+        and project.is_public
+        and (members[0].username == user.name or members[0].username == user.email)
+        and requesting_user_role is None
+    )
+
+    if not is_self_join_to_public:
+        if requesting_user_role not in [ProjectRole.ADMIN, ProjectRole.MANAGER]:
+            raise ForbiddenError("Access denied: insufficient permissions to add members")
+
+        if user.global_role != GlobalRole.ADMIN and requesting_user_role == ProjectRole.MANAGER:
+            for member in members:
+                if member.project_role == ProjectRole.ADMIN:
+                    raise ForbiddenError(
+                        "Access denied: only global admins can add project admins"
+                    )
+    else:
+        if members[0].project_role != ProjectRole.USER:
+            raise ForbiddenError("Access denied: can only join public projects as user role")
+
+    usernames = [member.username for member in members]
+
+    res = await session.execute(
+        select(UserModel).where((UserModel.name.in_(usernames)) | (UserModel.email.in_(usernames)))
+    )
+    users_found = res.scalars().all()
+
+    username_to_user = {user.name: user for user in users_found}
+    email_to_user = {user.email: user for user in users_found if user.email}
+
+    member_by_user_id = {m.user_id: m for m in project.members}
+
+    for member_setting in members:
+        user_to_add = username_to_user.get(member_setting.username) or email_to_user.get(
+            member_setting.username
+        )
+        if user_to_add is None:
+            raise ServerClientError(f"User not found: {member_setting.username}")
+
+        if user_to_add.id in member_by_user_id:
+            existing_member = member_by_user_id[user_to_add.id]
+            if existing_member.project_role != member_setting.project_role:
+                existing_member.project_role = member_setting.project_role
+        else:
+            await add_project_member(
+                session=session,
+                project=project,
+                user=user_to_add,
+                project_role=member_setting.project_role,
+                member_num=None,
+                commit=False,
+            )
+            member_by_user_id[user_to_add.id] = None
+
+    await session.commit()
+
+
 async def add_project_member(
     session: AsyncSession,
     project: ProjectModel,
@@ -497,8 +578,86 @@ def _is_project_admin(
     user: UserModel,
     project: ProjectModel,
 ) -> bool:
+    if user.id == project.owner_id:
+        return True
+
     for m in project.members:
         if user.id == m.user_id:
             if m.project_role == ProjectRole.ADMIN:
                 return True
     return False
+
+
+async def remove_project_members(
+    session: AsyncSession,
+    user: UserModel,
+    project: ProjectModel,
+    usernames: List[str],
+):
+    """Remove multiple members from a project."""
+    project = await get_project_model_by_name_or_error(
+        session=session,
+        project_name=project.name,
+    )
+    requesting_user_role = get_user_project_role(user=user, project=project)
+
+    is_self_leave = (
+        len(usernames) == 1
+        and (usernames[0] == user.name or usernames[0] == user.email)
+        and requesting_user_role is not None
+    )
+
+    if not is_self_leave:
+        if requesting_user_role not in [ProjectRole.ADMIN, ProjectRole.MANAGER]:
+            raise ForbiddenError("Access denied: insufficient permissions to remove members")
+
+    res = await session.execute(
+        select(UserModel).where((UserModel.name.in_(usernames)) | (UserModel.email.in_(usernames)))
+    )
+    users_found = res.scalars().all()
+
+    username_to_user = {user.name: user for user in users_found}
+    email_to_user = {user.email: user for user in users_found if user.email}
+
+    member_by_user_id = {m.user_id: m for m in project.members}
+
+    members_to_remove = []
+    admin_removals = 0
+
+    for username in usernames:
+        user_to_remove = username_to_user.get(username) or email_to_user.get(username)
+        if user_to_remove is None:
+            raise ServerClientError(f"User not found: {username}")
+
+        if user_to_remove.id not in member_by_user_id:
+            raise ServerClientError(f"User is not a member of this project: {username}")
+
+        member_to_remove = member_by_user_id[user_to_remove.id]
+
+        if member_to_remove.project_role == ProjectRole.ADMIN:
+            if is_self_leave:
+                total_admins = sum(
+                    1 for member in project.members if member.project_role == ProjectRole.ADMIN
+                )
+                if total_admins <= 1:
+                    raise ServerClientError("Cannot leave project: you are the last admin")
+            else:
+                if user.global_role != GlobalRole.ADMIN:
+                    raise ForbiddenError(
+                        f"Access denied: only global admins can remove project admins (user: {username})"
+                    )
+            admin_removals += 1
+
+        members_to_remove.append(member_to_remove)
+
+    if not is_self_leave:
+        total_admins = sum(
+            1 for member in project.members if member.project_role == ProjectRole.ADMIN
+        )
+        if admin_removals >= total_admins:
+            raise ServerClientError("Cannot remove all project admins")
+
+    for member in members_to_remove:
+        await session.delete(member)
+
+    await session.commit()

dstack/_internal/server/services/prometheus/client_metrics.py

@@ -0,0 +1,52 @@
+from prometheus_client import Counter, Histogram
+
+
+class RunMetrics:
+    """Wrapper class for run-related Prometheus metrics."""
+
+    def __init__(self):
+        self._submit_to_provision_duration = Histogram(
+            "dstack_submit_to_provision_duration_seconds",
+            "Time from when a run has been submitted and first job provisioning",
+            # Buckets optimized for percentile calculation
+            buckets=[
+                15,
+                30,
+                45,
+                60,
+                90,
+                120,
+                180,
+                240,
+                300,
+                360,
+                420,
+                480,
+                540,
+                600,
+                900,
+                1200,
+                1800,
+                float("inf"),
+            ],
+            labelnames=["project_name", "run_type"],
+        )
+
+        self._pending_runs_total = Counter(
+            "dstack_pending_runs_total",
+            "Number of pending runs",
+            labelnames=["project_name", "run_type"],
+        )
+
+    def log_submit_to_provision_duration(
+        self, duration_seconds: float, project_name: str, run_type: str
+    ):
+        self._submit_to_provision_duration.labels(
+            project_name=project_name, run_type=run_type
+        ).observe(duration_seconds)
+
+    def increment_pending_runs(self, project_name: str, run_type: str):
+        self._pending_runs_total.labels(project_name=project_name, run_type=run_type).inc()
+
+
+run_metrics = RunMetrics()

dstack/_internal/server/services/proxy/repo.py

@@ -7,6 +7,7 @@ from sqlalchemy.orm import joinedload
 
 import dstack._internal.server.services.jobs as jobs_services
 from dstack._internal.core.consts import DSTACK_RUNNER_SSH_PORT
+from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.configurations import ServiceConfiguration
 from dstack._internal.core.models.instances import RemoteConnectionInfo, SSHConnectionParams
 from dstack._internal.core.models.runs import (
@@ -86,6 +87,8 @@ class ServerProxyRepo(BaseProxyRepo):
                     username=jpd.username,
                     port=jpd.ssh_port,
                 )
+                if jpd.backend == BackendType.LOCAL:
+                    ssh_proxy = None
             ssh_head_proxy: Optional[SSHConnectionParams] = None
             ssh_head_proxy_private_key: Optional[str] = None
             instance = get_or_error(job.instance)

dstack/_internal/server/services/runner/client.py

@@ -109,6 +109,14 @@ class RunnerClient:
         )
         resp.raise_for_status()
 
+    def upload_archive(self, id: uuid.UUID, file: Union[BinaryIO, bytes]):
+        resp = requests.post(
+            self._url("/api/upload_archive"),
+            files={"archive": (str(id), file)},
+            timeout=UPLOAD_CODE_REQUEST_TIMEOUT,
+        )
+        resp.raise_for_status()
+
     def upload_code(self, file: Union[BinaryIO, bytes]):
         resp = requests.post(
             self._url("/api/upload_code"), data=file, timeout=UPLOAD_CODE_REQUEST_TIMEOUT