dstack 0.18.42__py3-none-any.whl → 0.18.44__py3-none-any.whl

dstack/_internal/server/routers/metrics.py

@@ -1,13 +1,16 @@
-from typing import Tuple
+from datetime import datetime
+from typing import Optional, Tuple
 
 from fastapi import APIRouter, Depends
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from dstack._internal.core.errors import ResourceNotExistsError
 from dstack._internal.core.models.metrics import JobMetrics
 from dstack._internal.server.db import get_session
 from dstack._internal.server.models import ProjectModel, UserModel
 from dstack._internal.server.security.permissions import ProjectMember
 from dstack._internal.server.services import metrics
+from dstack._internal.server.services.jobs import get_run_job_model
 from dstack._internal.server.utils.routers import get_base_api_additional_responses
 
 router = APIRouter(
@@ -24,6 +27,9 @@ async def get_job_metrics(
     run_name: str,
     replica_num: int = 0,
     job_num: int = 0,
+    limit: int = 1,
+    after: Optional[datetime] = None,
+    before: Optional[datetime] = None,
     session: AsyncSession = Depends(get_session),
     user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectMember()),
 ) -> JobMetrics:
@@ -31,6 +37,8 @@ async def get_job_metrics(
     Returns job-level metrics such as hardware utilization
     given `run_name`, `replica_num`, and `job_num`.
     If only `run_name` is specified, returns metrics of `(replica_num=0, job_num=0)`.
+    By default, returns one latest sample. To control time window/number of samples, use
+    `limit`, `after`, `before`.
 
     Supported metrics: [
         "cpu_usage_percent",
@@ -42,10 +50,21 @@ async def get_job_metrics(
     ]
     """
     _, project = user_project
-    return await metrics.get_job_metrics(
+
+    job_model = await get_run_job_model(
         session=session,
         project=project,
         run_name=run_name,
         replica_num=replica_num,
         job_num=job_num,
     )
+    if job_model is None:
+        raise ResourceNotExistsError("Found no job with given parameters")
+
+    return await metrics.get_job_metrics(
+        session=session,
+        job_model=job_model,
+        limit=limit,
+        after=after,
+        before=before,
+    )

dstack/_internal/server/routers/prometheus.py

@@ -0,0 +1,36 @@
+from typing import Annotated
+
+from fastapi import APIRouter, Depends
+from fastapi.responses import PlainTextResponse
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from dstack._internal.server import settings
+from dstack._internal.server.db import get_session
+from dstack._internal.server.deps import Project
+from dstack._internal.server.models import ProjectModel
+from dstack._internal.server.services import prometheus
+from dstack._internal.server.utils.routers import error_not_found
+
+router = APIRouter(
+    tags=["prometheus"],
+    default_response_class=PlainTextResponse,
+)
+
+
+@router.get("/metrics")
+async def get_prometheus_metrics(
+    session: Annotated[AsyncSession, Depends(get_session)],
+) -> str:
+    if not settings.ENABLE_PROMETHEUS_METRICS:
+        raise error_not_found()
+    return await prometheus.get_metrics(session=session)
+
+
+@router.get("/metrics/project/{project_name}")
+async def get_project_prometheus_metrics(
+    session: Annotated[AsyncSession, Depends(get_session)],
+    project: Annotated[ProjectModel, Depends(Project())],
+) -> str:
+    if not settings.ENABLE_PROMETHEUS_METRICS:
+        raise error_not_found()
+    return await prometheus.get_project_metrics(session=session, project=project)

dstack/_internal/server/security/permissions.py

@@ -80,7 +80,7 @@ class ProjectManager:
         project = await get_project_model_by_name(session=session, project_name=project_name)
         if project is None:
             raise error_forbidden()
-        if user.global_role in GlobalRole.ADMIN:
+        if user.global_role == GlobalRole.ADMIN:
             return user, project
         project_role = get_user_project_role(user=user, project=project)
         if project_role in [ProjectRole.ADMIN, ProjectRole.MANAGER]:

dstack/_internal/server/services/backends/configurators/aws.py

@@ -2,6 +2,7 @@ import concurrent.futures
 import json
 from typing import List
 
+import botocore.exceptions
 from boto3.session import Session
 
 from dstack._internal.core.backends.aws import AWSBackend, auth, compute, resources
@@ -35,6 +36,9 @@ from dstack._internal.server.services.backends.configurators.base import (
     Configurator,
     raise_invalid_credentials_error,
 )
+from dstack._internal.utils.logging import get_logger
+
+logger = get_logger(__name__)
 
 REGIONS = [
     ("US East, N. Virginia", "us-east-1"),
@@ -137,7 +141,8 @@ class AWSConfigurator(Configurator):
 
     def _check_config(self, session: Session, config: AWSConfigInfoWithCredsPartial):
         self._check_tags_config(config)
-        self._check_vpc_config(session=session, config=config)
+        self._check_iam_instance_profile_config(session, config)
+        self._check_vpc_config(session, config)
 
     def _check_tags_config(self, config: AWSConfigInfoWithCredsPartial):
         if not config.tags:
@@ -151,6 +156,31 @@ class AWSConfigurator(Configurator):
         except BackendError as e:
             raise ServerClientError(e.args[0])
 
+    def _check_iam_instance_profile_config(
+        self, session: Session, config: AWSConfigInfoWithCredsPartial
+    ):
+        if config.iam_instance_profile is None:
+            return
+        try:
+            iam_client = session.client("iam")
+            iam_client.get_instance_profile(InstanceProfileName=config.iam_instance_profile)
+        except botocore.exceptions.ClientError as e:
+            if e.response["Error"]["Code"] == "NoSuchEntity":
+                raise ServerClientError(
+                    f"IAM instance profile {config.iam_instance_profile} not found"
+                )
+            logger.exception(
+                "Got botocore.exceptions.ClientError when checking iam_instance_profile"
+            )
+            raise ServerClientError(
+                f"Failed to check IAM instance profile {config.iam_instance_profile}"
+            )
+        except Exception:
+            logger.exception("Got exception when checking iam_instance_profile")
+            raise ServerClientError(
+                f"Failed to check IAM instance profile {config.iam_instance_profile}"
+            )
+
     def _check_vpc_config(self, session: Session, config: AWSConfigInfoWithCredsPartial):
         allocate_public_ip = config.public_ips if config.public_ips is not None else True
         use_default_vpcs = config.default_vpcs if config.default_vpcs is not None else True

dstack/_internal/server/services/backends/configurators/gcp.py

@@ -127,10 +127,6 @@ class GCPConfigurator(Configurator):
             _, project_id = auth.authenticate(GCPDefaultCreds())
         except BackendAuthError:
             return []
-
-        if project_id is None:
-            return []
-
         return [
             GCPConfigInfoWithCreds(
                 project_id=project_id,
@@ -152,24 +148,21 @@ class GCPConfigurator(Configurator):
         ):
             raise_invalid_credentials_error(fields=[["creds"]])
         try:
-            credentials, project_id = auth.authenticate(creds=config.creds)
-        except BackendAuthError:
+            credentials, _ = auth.authenticate(creds=config.creds, project_id=config.project_id)
+        except BackendAuthError as e:
+            details = None
+            if len(e.args) > 0:
+                details = e.args[0]
             if is_core_model_instance(config.creds, GCPServiceAccountCreds):
-                raise_invalid_credentials_error(fields=[["creds", "data"]])
+                raise_invalid_credentials_error(fields=[["creds", "data"]], details=details)
             else:
-                raise_invalid_credentials_error(fields=[["creds"]])
-        if (
-            project_id is not None
-            and config.project_id is not None
-            and config.project_id != project_id
-        ):
-            raise ServerClientError(msg="Wrong project_id", fields=[["project_id"]])
-        config_values.project_id = self._get_project_id_element(selected=project_id)
+                raise_invalid_credentials_error(fields=[["creds"]], details=details)
         config_values.regions = self._get_regions_element(
             selected=config.regions or DEFAULT_REGIONS
         )
         if config.project_id is None:
             return config_values
+        config_values.project_id = self._get_project_id_element(selected=config.project_id)
         self._check_config(config=config, credentials=credentials)
         return config_values
 

dstack/_internal/server/services/backends/configurators/runpod.py

@@ -3,11 +3,7 @@ from typing import List
 
 from dstack._internal.core.backends.base import Backend
 from dstack._internal.core.backends.runpod import RunpodBackend, RunpodConfig, api_client
-from dstack._internal.core.models.backends.base import (
-    BackendType,
-    ConfigElementValue,
-    ConfigMultiElement,
-)
+from dstack._internal.core.models.backends.base import BackendType, ConfigMultiElement
 from dstack._internal.core.models.backends.runpod import (
     RunpodConfigInfo,
     RunpodConfigInfoWithCreds,
@@ -22,25 +18,6 @@ from dstack._internal.server.services.backends.configurators.base import (
     raise_invalid_credentials_error,
 )
 
-REGIONS = [
-    "CA-MTL-1",
-    "CA-MTL-2",
-    "CA-MTL-3",
-    "EU-NL-1",
-    "EU-RO-1",
-    "EU-SE-1",
-    "EUR-IS-1",
-    "EUR-IS-2",
-    "US-CA-1",
-    "US-GA-1",
-    "US-GA-2",
-    "US-KS-2",
-    "US-OR-1",
-    "US-TX-3",
-]
-
-DEFAULT_REGION = "CA-MTL-1"
-
 
 class RunpodConfigurator(Configurator):
     TYPE: BackendType = BackendType.RUNPOD
@@ -50,16 +27,12 @@ class RunpodConfigurator(Configurator):
         if config.creds is None:
             return config_values
         self._validate_runpod_api_key(config.creds.api_key)
-        config_values.regions = self._get_regions_element(
-            selected=config.regions or [DEFAULT_REGION]
-        )
+        config_values.regions = self._get_regions_element(selected=config.regions or [])
         return config_values
 
     def create_backend(
         self, project: ProjectModel, config: RunpodConfigInfoWithCreds
     ) -> BackendModel:
-        if config.regions is None:
-            config.regions = REGIONS
         return BackendModel(
             project_id=project.id,
             type=self.TYPE.value,
@@ -80,10 +53,7 @@ class RunpodConfigurator(Configurator):
         return RunpodBackend(config=config)
 
     def _get_regions_element(self, selected: List[str]) -> ConfigMultiElement:
-        element = ConfigMultiElement(selected=selected)
-        for r in REGIONS:
-            element.values.append(ConfigElementValue(value=r, label=r))
-        return element
+        return ConfigMultiElement(selected=selected)
 
     def _get_backend_config(self, model: BackendModel) -> RunpodConfig:
         return RunpodConfig(

dstack/_internal/server/services/config.py

@@ -6,6 +6,7 @@ from pydantic import BaseModel, Field, ValidationError, root_validator
 from sqlalchemy.ext.asyncio import AsyncSession
 from typing_extensions import Annotated
 
+from dstack._internal.core.backends.runpod.config import RUNPOD_COMMUNITY_CLOUD_DEFAULT
 from dstack._internal.core.errors import (
     BackendNotAvailable,
     ResourceNotExistsError,
@@ -45,7 +46,7 @@ logger = get_logger(__name__)
 # By default, PyYAML chooses the style of a collection depending on whether it has nested collections.
 # If a collection has nested collections, it will be assigned the block style. Otherwise it will have the flow style.
 #
-# We want mapping to always be display in block-style but lists without nested objects in flow-style.
+# We want mapping to always be displayed in block-style but lists without nested objects in flow-style.
 # So we define a custom representeter
 
 
@@ -107,6 +108,16 @@ class AWSConfig(CoreModel):
             )
         ),
     ] = None
+    iam_instance_profile: Annotated[
+        Optional[str],
+        Field(
+            description=(
+                "The name of the IAM instance profile to associate with EC2 instances."
+                " You can also specify the IAM role name for roles created via the AWS console."
+                " AWS automatically creates an instance profile and gives it the same name as the role"
+            )
+        ),
+    ] = None
     tags: Annotated[
         Optional[Dict[str, str]],
         Field(description="The tags that will be assigned to resources created by `dstack`"),
@@ -251,7 +262,7 @@ class GCPConfig(CoreModel):
         ),
     ] = None
     vm_service_account: Annotated[
-        Optional[str], Field(description="The service account associated with provisioned VMs")
+        Optional[str], Field(description="The service account to associate with provisioned VMs")
     ] = None
     tags: Annotated[
         Optional[Dict[str, str]],
@@ -330,7 +341,7 @@ class KubernetesConfig(CoreModel):
     kubeconfig: Annotated[KubeconfigConfig, Field(description="The kubeconfig configuration")]
     networking: Annotated[
         Optional[KubernetesNetworkingConfig], Field(description="The networking configuration")
-    ]
+    ] = None
 
 
 class KubernetesAPIConfig(CoreModel):
@@ -338,7 +349,7 @@ class KubernetesAPIConfig(CoreModel):
     kubeconfig: Annotated[KubeconfigAPIConfig, Field(description="The kubeconfig configuration")]
     networking: Annotated[
         Optional[KubernetesNetworkingConfig], Field(description="The networking configuration")
-    ]
+    ] = None
 
 
 class LambdaConfig(CoreModel):
@@ -418,6 +429,15 @@ class RunpodConfig(CoreModel):
         Optional[List[str]],
         Field(description="The list of RunPod regions. Omit to use all regions"),
     ] = None
+    community_cloud: Annotated[
+        Optional[bool],
+        Field(
+            description=(
+                "Whether Community Cloud offers can be suggested in addition to Secure Cloud."
+                f" Defaults to `{str(RUNPOD_COMMUNITY_CLOUD_DEFAULT).lower()}`"
+            )
+        ),
+    ] = None
     creds: Annotated[AnyRunpodCreds, Field(description="The credentials")]
 
 

dstack/_internal/server/services/fleets.py

@@ -517,6 +517,7 @@ async def delete_fleets(
             .options(selectinload(FleetModel.instances))
             .options(selectinload(FleetModel.runs))
             .execution_options(populate_existing=True)
+            .order_by(FleetModel.id)  # take locks in order
             .with_for_update()
         )
         fleet_models = res.scalars().unique().all()

dstack/_internal/server/services/gateways/__init__.py

@@ -220,6 +220,7 @@ async def delete_gateways(
             )
             .options(selectinload(GatewayModel.gateway_compute))
             .execution_options(populate_existing=True)
+            .order_by(GatewayModel.id)  # take locks in order
             .with_for_update()
         )
         gateway_models = res.scalars().all()

dstack/_internal/server/services/jobs/__init__.py

@@ -236,13 +236,14 @@ async def process_terminating_job(
         logger.debug("%s: stopping container", fmt(job_model))
         ssh_private_keys = get_instance_ssh_private_keys(instance_model)
         await stop_container(job_model, jpd, ssh_private_keys)
-        volume_models: list[VolumeModel]
         if jrd is not None and jrd.volume_names is not None:
-            volume_models = await list_project_volume_models(
-                session=session, project=instance_model.project, names=jrd.volume_names
-            )
+            volume_names = jrd.volume_names
         else:
-            volume_models = [va.volume for va in instance_model.volume_attachments]
+            # Legacy jobs before job_runtime_data/blocks were introduced
+            volume_names = [va.volume.name for va in instance_model.volume_attachments]
+        volume_models = await list_project_volume_models(
+            session=session, project=instance_model.project, names=volume_names
+        )
         if len(volume_models) > 0:
             logger.info("Detaching volumes: %s", [v.name for v in volume_models])
             all_volumes_detached = await _detach_volumes_from_job_instance(
@@ -302,11 +303,13 @@ async def process_volumes_detaching(
     jpd = get_or_error(get_job_provisioning_data(job_model))
     jrd = get_job_runtime_data(job_model)
     if jrd is not None and jrd.volume_names is not None:
-        volume_models = await list_project_volume_models(
-            session=session, project=instance_model.project, names=jrd.volume_names
-        )
+        volume_names = jrd.volume_names
     else:
-        volume_models = [va.volume for va in instance_model.volume_attachments]
+        # Legacy jobs before job_runtime_data/blocks were introduced
+        volume_names = [va.volume.name for va in instance_model.volume_attachments]
+    volume_models = await list_project_volume_models(
+        session=session, project=instance_model.project, names=volume_names
+    )
     logger.info("Detaching volumes: %s", [v.name for v in volume_models])
     all_volumes_detached = await _detach_volumes_from_job_instance(
         project=instance_model.project,

dstack/_internal/server/services/jobs/configurators/base.py

@@ -13,7 +13,11 @@ from dstack._internal.core.models.configurations import (
     PythonVersion,
     RunConfigurationType,
 )
-from dstack._internal.core.models.profiles import DEFAULT_STOP_DURATION, SpotPolicy
+from dstack._internal.core.models.profiles import (
+    DEFAULT_STOP_DURATION,
+    SpotPolicy,
+    UtilizationPolicy,
+)
 from dstack._internal.core.models.runs import (
     AppSpec,
     JobSpec,
@@ -113,6 +117,7 @@ class JobConfigurator(ABC):
             single_branch=self._single_branch(),
             max_duration=self._max_duration(),
             stop_duration=self._stop_duration(),
+            utilization_policy=self._utilization_policy(),
             registry_auth=self._registry_auth(),
             requirements=self._requirements(),
             retry=self._retry(),
@@ -201,6 +206,9 @@ class JobConfigurator(ABC):
         # pydantic validator ensures this is int
         return self.run_spec.merged_profile.stop_duration
 
+    def _utilization_policy(self) -> Optional[UtilizationPolicy]:
+        return self.run_spec.merged_profile.utilization_policy
+
     def _registry_auth(self) -> Optional[RegistryAuth]:
         return self.run_spec.configuration.registry_auth
 

dstack/_internal/server/services/jobs/configurators/dev.py

@@ -6,8 +6,6 @@ from dstack._internal.core.models.runs import RunSpec
 from dstack._internal.server.services.jobs.configurators.base import JobConfigurator
 from dstack._internal.server.services.jobs.configurators.extensions.vscode import VSCodeDesktop
 
-DEFAULT_MAX_DURATION_SECONDS = 6 * 3600
-
 INSTALL_IPYKERNEL = (
     "(echo pip install ipykernel... && pip install -q --no-cache-dir ipykernel 2> /dev/null) || "
     'echo "no pip, ipykernel was not installed"'
@@ -44,7 +42,7 @@ class DevEnvironmentJobConfigurator(JobConfigurator):
         return False
 
     def _default_max_duration(self) -> Optional[int]:
-        return DEFAULT_MAX_DURATION_SECONDS
+        return None
 
     def _spot_policy(self) -> SpotPolicy:
         return self.run_spec.merged_profile.spot_policy or SpotPolicy.ONDEMAND

dstack/_internal/server/services/jobs/configurators/task.py

@@ -5,8 +5,6 @@ from dstack._internal.core.models.profiles import SpotPolicy
 from dstack._internal.core.models.runs import JobSpec
 from dstack._internal.server.services.jobs.configurators.base import JobConfigurator
 
-DEFAULT_MAX_DURATION_SECONDS = 72 * 3600
-
 
 class TaskJobConfigurator(JobConfigurator):
     TYPE: RunConfigurationType = RunConfigurationType.TASK
@@ -29,7 +27,7 @@ class TaskJobConfigurator(JobConfigurator):
         return True
 
     def _default_max_duration(self) -> Optional[int]:
-        return DEFAULT_MAX_DURATION_SECONDS
+        return None
 
     def _spot_policy(self) -> SpotPolicy:
         return self.run_spec.merged_profile.spot_policy or SpotPolicy.ONDEMAND

dstack/_internal/server/services/logs/__init__.py

@@ -0,0 +1,78 @@
+import atexit
+from typing import List, Optional
+from uuid import UUID
+
+from dstack._internal.core.models.logs import JobSubmissionLogs
+from dstack._internal.server import settings
+from dstack._internal.server.models import ProjectModel
+from dstack._internal.server.schemas.logs import PollLogsRequest
+from dstack._internal.server.schemas.runner import LogEvent as RunnerLogEvent
+from dstack._internal.server.services.logs.aws import BOTO_AVAILABLE, CloudWatchLogStorage
+from dstack._internal.server.services.logs.base import LogStorage, LogStorageError
+from dstack._internal.server.services.logs.filelog import FileLogStorage
+from dstack._internal.server.services.logs.gcp import GCP_LOGGING_AVAILABLE, GCPLogStorage
+from dstack._internal.utils.common import run_async
+from dstack._internal.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+_log_storage: Optional[LogStorage] = None
+
+
+def get_log_storage() -> LogStorage:
+    global _log_storage
+    if _log_storage is not None:
+        return _log_storage
+    if settings.SERVER_CLOUDWATCH_LOG_GROUP:
+        if BOTO_AVAILABLE:
+            try:
+                _log_storage = CloudWatchLogStorage(
+                    group=settings.SERVER_CLOUDWATCH_LOG_GROUP,
+                    region=settings.SERVER_CLOUDWATCH_LOG_REGION,
+                )
+            except LogStorageError as e:
+                logger.error("Failed to initialize CloudWatch Logs storage: %s", e)
+            except Exception:
+                logger.exception("Got exception when initializing CloudWatch Logs storage")
+            else:
+                logger.debug("Using CloudWatch Logs storage")
+        else:
+            logger.error("Cannot use CloudWatch Logs storage: boto3 is not installed")
+    elif settings.SERVER_GCP_LOGGING_PROJECT:
+        if GCP_LOGGING_AVAILABLE:
+            try:
+                _log_storage = GCPLogStorage(project_id=settings.SERVER_GCP_LOGGING_PROJECT)
+            except LogStorageError as e:
+                logger.error("Failed to initialize GCP Logs storage: %s", e)
+            except Exception:
+                logger.exception("Got exception when initializing GCP Logs storage")
+            else:
+                logger.debug("Using GCP Logs storage")
+        else:
+            logger.error("Cannot use GCP Logs storage: GCP deps are not installed")
+    if _log_storage is None:
+        _log_storage = FileLogStorage()
+        logger.debug("Using file-based storage")
+    atexit.register(_log_storage.close)
+    return _log_storage
+
+
+def write_logs(
+    project: ProjectModel,
+    run_name: str,
+    job_submission_id: UUID,
+    runner_logs: List[RunnerLogEvent],
+    job_logs: List[RunnerLogEvent],
+) -> None:
+    return get_log_storage().write_logs(
+        project=project,
+        run_name=run_name,
+        job_submission_id=job_submission_id,
+        runner_logs=runner_logs,
+        job_logs=job_logs,
+    )
+
+
+async def poll_logs_async(project: ProjectModel, request: PollLogsRequest) -> JobSubmissionLogs:
+    return await run_async(get_log_storage().poll_logs, project=project, request=request)