dstack 0.18.43__py3-none-any.whl → 0.18.44__py3-none-any.whl

dstack/_internal/server/migrations/versions/60e444118b6d_add_jobprometheusmetrics.py

@@ -0,0 +1,40 @@
+"""Add JobPrometheusMetrics
+
+Revision ID: 60e444118b6d
+Revises: a751ef183f27
+Create Date: 2025-02-21 10:59:26.339353
+
+"""
+
+import sqlalchemy as sa
+import sqlalchemy_utils
+from alembic import op
+
+import dstack._internal.server.models
+
+# revision identifiers, used by Alembic.
+revision = "60e444118b6d"
+down_revision = "a751ef183f27"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "job_prometheus_metrics",
+        sa.Column("job_id", sqlalchemy_utils.types.uuid.UUIDType(binary=False), nullable=False),
+        sa.Column("collected_at", dstack._internal.server.models.NaiveDateTime(), nullable=False),
+        sa.Column("text", sa.Text(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["job_id"], ["jobs.id"], name=op.f("fk_job_prometheus_metrics_job_id_jobs")
+        ),
+        sa.PrimaryKeyConstraint("job_id", name=op.f("pk_job_prometheus_metrics")),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("job_prometheus_metrics")
+    # ### end Alembic commands ###

dstack/_internal/server/migrations/versions/98d1b92988bc_add_jobterminationreason_terminated_due_.py

@@ -0,0 +1,140 @@
+"""Add JobTerminationReason.TERMINATED_DUE_TO_UTILIZATION_POLICY
+
+Revision ID: 98d1b92988bc
+Revises: 60e444118b6d
+Create Date: 2025-02-28 15:12:37.649876
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from alembic_postgresql_enum import TableReference
+
+# revision identifiers, used by Alembic.
+revision = "98d1b92988bc"
+down_revision = "60e444118b6d"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # SQLite
+    with op.batch_alter_table("jobs", schema=None) as batch_op:
+        batch_op.alter_column(
+            "termination_reason",
+            existing_type=sa.VARCHAR(length=34),
+            type_=sa.Enum(
+                "FAILED_TO_START_DUE_TO_NO_CAPACITY",
+                "INTERRUPTED_BY_NO_CAPACITY",
+                "WAITING_INSTANCE_LIMIT_EXCEEDED",
+                "WAITING_RUNNER_LIMIT_EXCEEDED",
+                "TERMINATED_BY_USER",
+                "VOLUME_ERROR",
+                "GATEWAY_ERROR",
+                "SCALED_DOWN",
+                "DONE_BY_RUNNER",
+                "ABORTED_BY_USER",
+                "TERMINATED_BY_SERVER",
+                "INACTIVITY_DURATION_EXCEEDED",
+                "TERMINATED_DUE_TO_UTILIZATION_POLICY",
+                "CONTAINER_EXITED_WITH_ERROR",
+                "PORTS_BINDING_FAILED",
+                "CREATING_CONTAINER_ERROR",
+                "EXECUTOR_ERROR",
+                "MAX_DURATION_EXCEEDED",
+                name="jobterminationreason",
+            ),
+            existing_nullable=True,
+        )
+    # PostgreSQL
+    op.sync_enum_values(
+        enum_schema="public",
+        enum_name="jobterminationreason",
+        new_values=[
+            "FAILED_TO_START_DUE_TO_NO_CAPACITY",
+            "INTERRUPTED_BY_NO_CAPACITY",
+            "WAITING_INSTANCE_LIMIT_EXCEEDED",
+            "WAITING_RUNNER_LIMIT_EXCEEDED",
+            "TERMINATED_BY_USER",
+            "VOLUME_ERROR",
+            "GATEWAY_ERROR",
+            "SCALED_DOWN",
+            "DONE_BY_RUNNER",
+            "ABORTED_BY_USER",
+            "TERMINATED_BY_SERVER",
+            "INACTIVITY_DURATION_EXCEEDED",
+            "TERMINATED_DUE_TO_UTILIZATION_POLICY",
+            "CONTAINER_EXITED_WITH_ERROR",
+            "PORTS_BINDING_FAILED",
+            "CREATING_CONTAINER_ERROR",
+            "EXECUTOR_ERROR",
+            "MAX_DURATION_EXCEEDED",
+        ],
+        affected_columns=[
+            TableReference(
+                table_schema="public", table_name="jobs", column_name="termination_reason"
+            )
+        ],
+        enum_values_to_rename=[],
+    )
+
+
+def downgrade() -> None:
+    # SQLite
+    with op.batch_alter_table("jobs", schema=None) as batch_op:
+        batch_op.alter_column(
+            "termination_reason",
+            existing_type=sa.Enum(
+                "FAILED_TO_START_DUE_TO_NO_CAPACITY",
+                "INTERRUPTED_BY_NO_CAPACITY",
+                "WAITING_INSTANCE_LIMIT_EXCEEDED",
+                "WAITING_RUNNER_LIMIT_EXCEEDED",
+                "TERMINATED_BY_USER",
+                "VOLUME_ERROR",
+                "GATEWAY_ERROR",
+                "SCALED_DOWN",
+                "DONE_BY_RUNNER",
+                "ABORTED_BY_USER",
+                "TERMINATED_BY_SERVER",
+                "INACTIVITY_DURATION_EXCEEDED",
+                "TERMINATED_DUE_TO_UTILIZATION_POLICY",
+                "CONTAINER_EXITED_WITH_ERROR",
+                "PORTS_BINDING_FAILED",
+                "CREATING_CONTAINER_ERROR",
+                "EXECUTOR_ERROR",
+                "MAX_DURATION_EXCEEDED",
+                name="jobterminationreason",
+            ),
+            type_=sa.VARCHAR(length=34),
+            existing_nullable=True,
+        )
+    # PostgreSQL
+    op.sync_enum_values(
+        enum_schema="public",
+        enum_name="jobterminationreason",
+        new_values=[
+            "FAILED_TO_START_DUE_TO_NO_CAPACITY",
+            "INTERRUPTED_BY_NO_CAPACITY",
+            "WAITING_INSTANCE_LIMIT_EXCEEDED",
+            "WAITING_RUNNER_LIMIT_EXCEEDED",
+            "TERMINATED_BY_USER",
+            "VOLUME_ERROR",
+            "GATEWAY_ERROR",
+            "SCALED_DOWN",
+            "DONE_BY_RUNNER",
+            "ABORTED_BY_USER",
+            "TERMINATED_BY_SERVER",
+            "INACTIVITY_DURATION_EXCEEDED",
+            "CONTAINER_EXITED_WITH_ERROR",
+            "PORTS_BINDING_FAILED",
+            "CREATING_CONTAINER_ERROR",
+            "EXECUTOR_ERROR",
+            "MAX_DURATION_EXCEEDED",
+        ],
+        affected_columns=[
+            TableReference(
+                table_schema="public", table_name="jobs", column_name="termination_reason"
+            )
+        ],
+        enum_values_to_rename=[],
+    )

dstack/_internal/server/models.py

@@ -648,3 +648,14 @@ class JobMetricsPoint(BaseModel):
     # json-encoded lists of metric values of len(gpus) length
     gpus_memory_usage_bytes: Mapped[str] = mapped_column(Text)
     gpus_util_percent: Mapped[str] = mapped_column(Text)
+
+
+class JobPrometheusMetrics(BaseModel):
+    __tablename__ = "job_prometheus_metrics"
+
+    job_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("jobs.id"), primary_key=True)
+    job: Mapped["JobModel"] = relationship()
+
+    collected_at: Mapped[datetime] = mapped_column(NaiveDateTime)
+    # Raw Prometheus text response
+    text: Mapped[str] = mapped_column(Text)

dstack/_internal/server/routers/metrics.py

@@ -1,13 +1,16 @@
-from typing import Tuple
+from datetime import datetime
+from typing import Optional, Tuple
 
 from fastapi import APIRouter, Depends
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from dstack._internal.core.errors import ResourceNotExistsError
 from dstack._internal.core.models.metrics import JobMetrics
 from dstack._internal.server.db import get_session
 from dstack._internal.server.models import ProjectModel, UserModel
 from dstack._internal.server.security.permissions import ProjectMember
 from dstack._internal.server.services import metrics
+from dstack._internal.server.services.jobs import get_run_job_model
 from dstack._internal.server.utils.routers import get_base_api_additional_responses
 
 router = APIRouter(
@@ -24,6 +27,9 @@ async def get_job_metrics(
     run_name: str,
     replica_num: int = 0,
     job_num: int = 0,
+    limit: int = 1,
+    after: Optional[datetime] = None,
+    before: Optional[datetime] = None,
     session: AsyncSession = Depends(get_session),
     user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectMember()),
 ) -> JobMetrics:
@@ -31,6 +37,8 @@ async def get_job_metrics(
     Returns job-level metrics such as hardware utilization
     given `run_name`, `replica_num`, and `job_num`.
     If only `run_name` is specified, returns metrics of `(replica_num=0, job_num=0)`.
+    By default, returns one latest sample. To control time window/number of samples, use
+    `limit`, `after`, `before`.
 
     Supported metrics: [
         "cpu_usage_percent",
@@ -42,10 +50,21 @@ async def get_job_metrics(
     ]
     """
     _, project = user_project
-    return await metrics.get_job_metrics(
+
+    job_model = await get_run_job_model(
         session=session,
         project=project,
         run_name=run_name,
         replica_num=replica_num,
         job_num=job_num,
     )
+    if job_model is None:
+        raise ResourceNotExistsError("Found no job with given parameters")
+
+    return await metrics.get_job_metrics(
+        session=session,
+        job_model=job_model,
+        limit=limit,
+        after=after,
+        before=before,
+    )

dstack/_internal/server/routers/prometheus.py

@@ -0,0 +1,36 @@
+from typing import Annotated
+
+from fastapi import APIRouter, Depends
+from fastapi.responses import PlainTextResponse
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from dstack._internal.server import settings
+from dstack._internal.server.db import get_session
+from dstack._internal.server.deps import Project
+from dstack._internal.server.models import ProjectModel
+from dstack._internal.server.services import prometheus
+from dstack._internal.server.utils.routers import error_not_found
+
+router = APIRouter(
+    tags=["prometheus"],
+    default_response_class=PlainTextResponse,
+)
+
+
+@router.get("/metrics")
+async def get_prometheus_metrics(
+    session: Annotated[AsyncSession, Depends(get_session)],
+) -> str:
+    if not settings.ENABLE_PROMETHEUS_METRICS:
+        raise error_not_found()
+    return await prometheus.get_metrics(session=session)
+
+
+@router.get("/metrics/project/{project_name}")
+async def get_project_prometheus_metrics(
+    session: Annotated[AsyncSession, Depends(get_session)],
+    project: Annotated[ProjectModel, Depends(Project())],
+) -> str:
+    if not settings.ENABLE_PROMETHEUS_METRICS:
+        raise error_not_found()
+    return await prometheus.get_project_metrics(session=session, project=project)

dstack/_internal/server/security/permissions.py

@@ -80,7 +80,7 @@ class ProjectManager:
         project = await get_project_model_by_name(session=session, project_name=project_name)
         if project is None:
             raise error_forbidden()
-        if user.global_role in GlobalRole.ADMIN:
+        if user.global_role == GlobalRole.ADMIN:
             return user, project
         project_role = get_user_project_role(user=user, project=project)
         if project_role in [ProjectRole.ADMIN, ProjectRole.MANAGER]:

dstack/_internal/server/services/backends/configurators/runpod.py

@@ -3,11 +3,7 @@ from typing import List
 
 from dstack._internal.core.backends.base import Backend
 from dstack._internal.core.backends.runpod import RunpodBackend, RunpodConfig, api_client
-from dstack._internal.core.models.backends.base import (
-    BackendType,
-    ConfigElementValue,
-    ConfigMultiElement,
-)
+from dstack._internal.core.models.backends.base import BackendType, ConfigMultiElement
 from dstack._internal.core.models.backends.runpod import (
     RunpodConfigInfo,
     RunpodConfigInfoWithCreds,
@@ -22,25 +18,6 @@ from dstack._internal.server.services.backends.configurators.base import (
     raise_invalid_credentials_error,
 )
 
-REGIONS = [
-    "CA-MTL-1",
-    "CA-MTL-2",
-    "CA-MTL-3",
-    "EU-NL-1",
-    "EU-RO-1",
-    "EU-SE-1",
-    "EUR-IS-1",
-    "EUR-IS-2",
-    "US-CA-1",
-    "US-GA-1",
-    "US-GA-2",
-    "US-KS-2",
-    "US-OR-1",
-    "US-TX-3",
-]
-
-DEFAULT_REGION = "CA-MTL-1"
-
 
 class RunpodConfigurator(Configurator):
     TYPE: BackendType = BackendType.RUNPOD
@@ -50,16 +27,12 @@ class RunpodConfigurator(Configurator):
         if config.creds is None:
             return config_values
         self._validate_runpod_api_key(config.creds.api_key)
-        config_values.regions = self._get_regions_element(
-            selected=config.regions or [DEFAULT_REGION]
-        )
+        config_values.regions = self._get_regions_element(selected=config.regions or [])
         return config_values
 
     def create_backend(
         self, project: ProjectModel, config: RunpodConfigInfoWithCreds
     ) -> BackendModel:
-        if config.regions is None:
-            config.regions = REGIONS
         return BackendModel(
             project_id=project.id,
             type=self.TYPE.value,
@@ -80,10 +53,7 @@ class RunpodConfigurator(Configurator):
         return RunpodBackend(config=config)
 
     def _get_regions_element(self, selected: List[str]) -> ConfigMultiElement:
-        element = ConfigMultiElement(selected=selected)
-        for r in REGIONS:
-            element.values.append(ConfigElementValue(value=r, label=r))
-        return element
+        return ConfigMultiElement(selected=selected)
 
     def _get_backend_config(self, model: BackendModel) -> RunpodConfig:
         return RunpodConfig(

dstack/_internal/server/services/config.py

@@ -6,6 +6,7 @@ from pydantic import BaseModel, Field, ValidationError, root_validator
 from sqlalchemy.ext.asyncio import AsyncSession
 from typing_extensions import Annotated
 
+from dstack._internal.core.backends.runpod.config import RUNPOD_COMMUNITY_CLOUD_DEFAULT
 from dstack._internal.core.errors import (
     BackendNotAvailable,
     ResourceNotExistsError,
@@ -45,7 +46,7 @@ logger = get_logger(__name__)
 # By default, PyYAML chooses the style of a collection depending on whether it has nested collections.
 # If a collection has nested collections, it will be assigned the block style. Otherwise it will have the flow style.
 #
-# We want mapping to always be display in block-style but lists without nested objects in flow-style.
+# We want mapping to always be displayed in block-style but lists without nested objects in flow-style.
 # So we define a custom representeter
 
 
@@ -340,7 +341,7 @@ class KubernetesConfig(CoreModel):
     kubeconfig: Annotated[KubeconfigConfig, Field(description="The kubeconfig configuration")]
     networking: Annotated[
         Optional[KubernetesNetworkingConfig], Field(description="The networking configuration")
-    ]
+    ] = None
 
 
 class KubernetesAPIConfig(CoreModel):
@@ -348,7 +349,7 @@ class KubernetesAPIConfig(CoreModel):
     kubeconfig: Annotated[KubeconfigAPIConfig, Field(description="The kubeconfig configuration")]
     networking: Annotated[
         Optional[KubernetesNetworkingConfig], Field(description="The networking configuration")
-    ]
+    ] = None
 
 
 class LambdaConfig(CoreModel):
@@ -428,6 +429,15 @@ class RunpodConfig(CoreModel):
         Optional[List[str]],
         Field(description="The list of RunPod regions. Omit to use all regions"),
     ] = None
+    community_cloud: Annotated[
+        Optional[bool],
+        Field(
+            description=(
+                "Whether Community Cloud offers can be suggested in addition to Secure Cloud."
+                f" Defaults to `{str(RUNPOD_COMMUNITY_CLOUD_DEFAULT).lower()}`"
+            )
+        ),
+    ] = None
     creds: Annotated[AnyRunpodCreds, Field(description="The credentials")]
 
 

dstack/_internal/server/services/fleets.py

@@ -517,6 +517,7 @@ async def delete_fleets(
             .options(selectinload(FleetModel.instances))
             .options(selectinload(FleetModel.runs))
             .execution_options(populate_existing=True)
+            .order_by(FleetModel.id)  # take locks in order
             .with_for_update()
         )
         fleet_models = res.scalars().unique().all()

dstack/_internal/server/services/gateways/__init__.py

@@ -220,6 +220,7 @@ async def delete_gateways(
             )
             .options(selectinload(GatewayModel.gateway_compute))
             .execution_options(populate_existing=True)
+            .order_by(GatewayModel.id)  # take locks in order
             .with_for_update()
         )
         gateway_models = res.scalars().all()

dstack/_internal/server/services/jobs/configurators/base.py

@@ -13,7 +13,11 @@ from dstack._internal.core.models.configurations import (
     PythonVersion,
     RunConfigurationType,
 )
-from dstack._internal.core.models.profiles import DEFAULT_STOP_DURATION, SpotPolicy
+from dstack._internal.core.models.profiles import (
+    DEFAULT_STOP_DURATION,
+    SpotPolicy,
+    UtilizationPolicy,
+)
 from dstack._internal.core.models.runs import (
     AppSpec,
     JobSpec,
@@ -113,6 +117,7 @@ class JobConfigurator(ABC):
             single_branch=self._single_branch(),
             max_duration=self._max_duration(),
             stop_duration=self._stop_duration(),
+            utilization_policy=self._utilization_policy(),
             registry_auth=self._registry_auth(),
             requirements=self._requirements(),
             retry=self._retry(),
@@ -201,6 +206,9 @@ class JobConfigurator(ABC):
         # pydantic validator ensures this is int
         return self.run_spec.merged_profile.stop_duration
 
+    def _utilization_policy(self) -> Optional[UtilizationPolicy]:
+        return self.run_spec.merged_profile.utilization_policy
+
     def _registry_auth(self) -> Optional[RegistryAuth]:
         return self.run_spec.configuration.registry_auth