dstack 0.0.9__py3-none-any.whl → 0.20.7__py3-none-any.whl

dstack/_internal/server/services/prometheus/client_metrics.py

@@ -0,0 +1,55 @@
+from prometheus_client import Counter, Histogram
+
+
+class RunMetrics:
+    """Wrapper class for run-related Prometheus metrics."""
+
+    def __init__(self):
+        # submit_to_provision_duration reflects real provisioning time
+        # but does not reflect how quickly provisioning processing works
+        # since it includes scheduling time, retrying, etc.
+        self._submit_to_provision_duration = Histogram(
+            "dstack_submit_to_provision_duration_seconds",
+            "Time from when a run has been submitted and first job provisioning",
+            # Buckets optimized for percentile calculation
+            buckets=[
+                15,
+                30,
+                45,
+                60,
+                90,
+                120,
+                180,
+                240,
+                300,
+                360,
+                420,
+                480,
+                540,
+                600,
+                900,
+                1200,
+                1800,
+                float("inf"),
+            ],
+            labelnames=["project_name", "run_type"],
+        )
+
+        self._pending_runs_total = Counter(
+            "dstack_pending_runs_total",
+            "Number of pending runs",
+            labelnames=["project_name", "run_type"],
+        )
+
+    def log_submit_to_provision_duration(
+        self, duration_seconds: float, project_name: str, run_type: str
+    ):
+        self._submit_to_provision_duration.labels(
+            project_name=project_name, run_type=run_type
+        ).observe(duration_seconds)
+
+    def increment_pending_runs(self, project_name: str, run_type: str):
+        self._pending_runs_total.labels(project_name=project_name, run_type=run_type).inc()
+
+
+run_metrics = RunMetrics()

dstack/_internal/server/services/prometheus/custom_metrics.py

@@ -0,0 +1,327 @@
+import itertools
+import json
+from collections import defaultdict
+from collections.abc import Generator, Iterable
+from typing import ClassVar
+from uuid import UUID
+
+from prometheus_client import Metric
+from prometheus_client.parser import text_string_to_metric_families
+from prometheus_client.samples import Sample
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import aliased, joinedload
+
+from dstack._internal.core.models.instances import InstanceStatus
+from dstack._internal.core.models.runs import JobStatus, RunSpec, RunStatus
+from dstack._internal.server.models import (
+    InstanceModel,
+    JobMetricsPoint,
+    JobModel,
+    JobPrometheusMetrics,
+    ProjectModel,
+    RunModel,
+    UserModel,
+)
+from dstack._internal.server.services.instances import get_instance_offer
+from dstack._internal.server.services.jobs import get_job_provisioning_data, get_job_runtime_data
+from dstack._internal.utils.common import get_current_datetime
+
+
+async def get_metrics(session: AsyncSession) -> str:
+    metrics_iter = itertools.chain(
+        await get_instance_metrics(session),
+        await get_run_metrics(session),
+        await get_job_metrics(session),
+    )
+    return "\n".join(_render_metrics(metrics_iter)) + "\n"
+
+
+async def get_instance_metrics(session: AsyncSession) -> Iterable[Metric]:
+    res = await session.execute(
+        select(InstanceModel)
+        .join(ProjectModel)
+        .where(
+            InstanceModel.deleted == False,
+            InstanceModel.status.in_(
+                [
+                    InstanceStatus.PROVISIONING,
+                    InstanceStatus.IDLE,
+                    InstanceStatus.BUSY,
+                    InstanceStatus.TERMINATING,
+                ]
+            ),
+        )
+        .order_by(ProjectModel.name, InstanceModel.name)
+        .options(
+            joinedload(InstanceModel.project),
+            joinedload(InstanceModel.fleet),
+        )
+    )
+    instances = res.unique().scalars().all()
+    metrics = _InstanceMetrics()
+    now = get_current_datetime()
+    for instance in instances:
+        fleet = instance.fleet
+        offer = get_instance_offer(instance)
+        gpu = ""
+        gpu_count = 0
+        if offer is not None and len(offer.instance.resources.gpus) > 0:
+            gpu = offer.instance.resources.gpus[0].name
+            gpu_count = len(offer.instance.resources.gpus)
+        labels: dict[str, str] = {
+            "dstack_project_name": instance.project.name,
+            "dstack_fleet_name": fleet.name if fleet is not None else "",
+            "dstack_fleet_id": str(fleet.id) if fleet is not None else "",
+            "dstack_instance_name": str(instance.name),
+            "dstack_instance_id": str(instance.id),
+            "dstack_instance_type": offer.instance.name if offer is not None else "",
+            "dstack_backend": instance.backend.value if instance.backend is not None else "",
+            "dstack_gpu": gpu,
+        }
+        duration = (now - instance.created_at).total_seconds()
+        metrics.add_sample(_INSTANCE_DURATION, labels, duration)
+        metrics.add_sample(_INSTANCE_PRICE, labels, instance.price or 0.0)
+        metrics.add_sample(_INSTANCE_GPU_COUNT, labels, gpu_count)
+    return metrics.values()
+
+
+async def get_run_metrics(session: AsyncSession) -> Iterable[Metric]:
+    res = await session.execute(
+        select(ProjectModel.name, UserModel.name, RunModel.status, func.count(RunModel.id))
+        .join_from(RunModel, ProjectModel)
+        .join_from(RunModel, UserModel, RunModel.user_id == UserModel.id)
+        .group_by(ProjectModel.name, UserModel.name, RunModel.status)
+        .order_by(ProjectModel.name, UserModel.name, RunModel.status)
+    )
+    projects: dict[str, dict[str, dict[RunStatus, int]]] = defaultdict(
+        lambda: defaultdict(lambda: defaultdict(int))
+    )
+    for project_name, user_name, status, count in res.all():
+        projects[project_name][user_name][status] = count
+    metrics = _RunMetrics()
+    for project_name, users in projects.items():
+        for user_name, statuses in users.items():
+            labels: dict[str, str] = {
+                "dstack_project_name": project_name,
+                "dstack_user_name": user_name,
+            }
+            metrics.add_sample(_RUN_COUNT_TOTAL, labels, sum(statuses.values()))
+            metrics.add_sample(_RUN_COUNT_TERMINATED, labels, statuses[RunStatus.TERMINATED])
+            metrics.add_sample(_RUN_COUNT_FAILED, labels, statuses[RunStatus.FAILED])
+            metrics.add_sample(_RUN_COUNT_DONE, labels, statuses[RunStatus.DONE])
+    return metrics.values()
+
+
+async def get_job_metrics(session: AsyncSession) -> Iterable[Metric]:
+    res = await session.execute(
+        select(JobModel)
+        .join(ProjectModel)
+        .where(
+            JobModel.status.in_(
+                [
+                    JobStatus.PROVISIONING,
+                    JobStatus.PULLING,
+                    JobStatus.RUNNING,
+                    JobStatus.TERMINATING,
+                ]
+            )
+        )
+        .order_by(ProjectModel.name, JobModel.job_name)
+        .options(
+            joinedload(JobModel.project),
+            joinedload(JobModel.run).joinedload(RunModel.user),
+        )
+    )
+    jobs = res.scalars().all()
+    job_ids = {job.id for job in jobs}
+    job_metrics_points = await _get_job_metrics_points(session, job_ids)
+    job_prometheus_metrics = await _get_job_prometheus_metrics(session, job_ids)
+
+    metrics = _JobMetrics()
+    now = get_current_datetime()
+    for job in jobs:
+        jpd = get_job_provisioning_data(job)
+        if jpd is None:
+            continue
+        jrd = get_job_runtime_data(job)
+        resources = jpd.instance_type.resources
+        price = jpd.price
+        if jrd is not None and jrd.offer is not None:
+            resources = jrd.offer.instance.resources
+            price = jrd.offer.price
+        gpus = resources.gpus
+        cpus = resources.cpus
+        run_spec = RunSpec.__response__.parse_raw(job.run.run_spec)
+        labels = {
+            "dstack_project_name": job.project.name,
+            "dstack_user_name": job.run.user.name,
+            "dstack_run_name": job.run_name,
+            "dstack_run_id": str(job.run_id),
+            "dstack_job_name": job.job_name,
+            "dstack_job_id": str(job.id),
+            "dstack_job_num": str(job.job_num),
+            "dstack_replica_num": str(job.replica_num),
+            "dstack_run_type": run_spec.configuration.type,
+            "dstack_backend": jpd.get_base_backend().value,
+            "dstack_gpu": gpus[0].name if gpus else "",
+        }
+        duration = (now - job.submitted_at).total_seconds()
+        metrics.add_sample(_JOB_DURATION, labels, duration)
+        metrics.add_sample(_JOB_PRICE, labels, price)
+        metrics.add_sample(_JOB_GPU_COUNT, labels, len(gpus))
+        metrics.add_sample(_JOB_CPU_COUNT, labels, cpus)
+        metrics.add_sample(_JOB_MEMORY_TOTAL, labels, resources.memory_mib * 1024 * 1024)
+        jmp = job_metrics_points.get(job.id)
+        if jmp is not None:
+            metrics.add_sample(_JOB_CPU_TIME, labels, jmp.cpu_usage_micro / 1_000_000)
+            metrics.add_sample(_JOB_MEMORY_USAGE, labels, jmp.memory_usage_bytes)
+            metrics.add_sample(_JOB_MEMORY_WORKING_SET, labels, jmp.memory_working_set_bytes)
+            if gpus:
+                gpu_memory_total = gpus[0].memory_mib * 1024 * 1024
+                for gpu_num, (gpu_util, gpu_memory_usage) in enumerate(
+                    zip(
+                        json.loads(jmp.gpus_util_percent),
+                        json.loads(jmp.gpus_memory_usage_bytes),
+                    )
+                ):
+                    gpu_labels = labels.copy()
+                    gpu_labels["dstack_gpu_num"] = gpu_num
+                    metrics.add_sample(_JOB_GPU_USAGE_RATIO, gpu_labels, gpu_util / 100)
+                    metrics.add_sample(_JOB_GPU_MEMORY_TOTAL, gpu_labels, gpu_memory_total)
+                    metrics.add_sample(_JOB_GPU_MEMORY_USAGE, gpu_labels, gpu_memory_usage)
+        jpm = job_prometheus_metrics.get(job.id)
+        if jpm is not None:
+            for metric in text_string_to_metric_families(jpm.text):
+                metrics.add_metric(metric, labels)
+    return metrics.values()
+
+
+_COUNTER = "counter"
+_GAUGE = "gauge"
+
+_INSTANCE_DURATION = "dstack_instance_duration_seconds_total"
+_INSTANCE_PRICE = "dstack_instance_price_dollars_per_hour"
+_INSTANCE_GPU_COUNT = "dstack_instance_gpu_count"
+_RUN_COUNT_TOTAL = "dstack_run_count_total"
+_RUN_COUNT_TERMINATED = "dstack_run_count_terminated_total"
+_RUN_COUNT_FAILED = "dstack_run_count_failed_total"
+_RUN_COUNT_DONE = "dstack_run_count_done_total"
+_JOB_DURATION = "dstack_job_duration_seconds_total"
+_JOB_PRICE = "dstack_job_price_dollars_per_hour"
+_JOB_GPU_COUNT = "dstack_job_gpu_count"
+_JOB_CPU_COUNT = "dstack_job_cpu_count"
+_JOB_CPU_TIME = "dstack_job_cpu_time_seconds_total"
+_JOB_MEMORY_TOTAL = "dstack_job_memory_total_bytes"
+_JOB_MEMORY_USAGE = "dstack_job_memory_usage_bytes"
+_JOB_MEMORY_WORKING_SET = "dstack_job_memory_working_set_bytes"
+_JOB_GPU_USAGE_RATIO = "dstack_job_gpu_usage_ratio"
+_JOB_GPU_MEMORY_TOTAL = "dstack_job_gpu_memory_total_bytes"
+_JOB_GPU_MEMORY_USAGE = "dstack_job_gpu_memory_usage_bytes"
+
+
+class _Metrics(dict[str, Metric]):
+    metrics: ClassVar[list[tuple[str, str, str]]]
+
+    def __init__(self):
+        super().__init__()
+        for name, typ, documentation in self.metrics:
+            self[name] = Metric(name=name, documentation=documentation, typ=typ)
+
+    def add_sample(self, name: str, labels: dict[str, str], value: float) -> None:
+        # NOTE: Keeps reference to labels.
+        self[name].add_sample(name=name, labels=labels, value=value)
+
+    def add_metric(self, metric: Metric, labels: dict[str, str]) -> None:
+        # NOTE: Modifies and keeps reference to metric.
+        name = metric.name
+        samples = metric.samples
+        stored_metric = self.get(name)
+        if stored_metric is None:
+            stored_metric = metric
+            stored_metric.samples = []
+            self[name] = stored_metric
+        for sample in samples:
+            sample.labels.update(labels)
+            # text_string_to_metric_families "fixes" counter names appending _total,
+            # we rebuild Sample to revert this
+            stored_metric.samples.append(Sample(name, *sample[1:]))
+
+
+class _InstanceMetrics(_Metrics):
+    metrics = [
+        (_INSTANCE_DURATION, _COUNTER, "Total seconds the instance is running"),
+        (_INSTANCE_PRICE, _GAUGE, "Instance price, USD/hour"),
+        (_INSTANCE_GPU_COUNT, _GAUGE, "Instance GPU count"),
+    ]
+
+
+class _RunMetrics(_Metrics):
+    metrics = [
+        (_RUN_COUNT_TOTAL, _COUNTER, "Total runs count"),
+        (_RUN_COUNT_TERMINATED, _COUNTER, "Terminated runs count"),
+        (_RUN_COUNT_FAILED, _COUNTER, "Failed runs count"),
+        (_RUN_COUNT_DONE, _COUNTER, "Done runs count"),
+    ]
+
+
+class _JobMetrics(_Metrics):
+    metrics = [
+        (_JOB_DURATION, _COUNTER, "Total seconds the job is running"),
+        (_JOB_PRICE, _GAUGE, "Job instance price, USD/hour"),
+        (_JOB_GPU_COUNT, _GAUGE, "Job GPU count"),
+        (_JOB_CPU_COUNT, _GAUGE, "Job CPU count"),
+        (_JOB_CPU_TIME, _COUNTER, "Total CPU time consumed by the job, seconds"),
+        (_JOB_MEMORY_TOTAL, _GAUGE, "Total memory allocated for the job, bytes"),
+        (_JOB_MEMORY_USAGE, _GAUGE, "Memory used by the job (including cache), bytes"),
+        (_JOB_MEMORY_WORKING_SET, _GAUGE, "Memory used by the job (not including cache), bytes"),
+        (_JOB_GPU_USAGE_RATIO, _GAUGE, "Job GPU usage, percent (as 0.0-1.0)"),
+        (_JOB_GPU_MEMORY_TOTAL, _GAUGE, "Total GPU memory allocated for the job, bytes"),
+        (_JOB_GPU_MEMORY_USAGE, _GAUGE, "GPU memory used by the job, bytes"),
+    ]
+
+
+async def _get_job_metrics_points(
+    session: AsyncSession, job_ids: Iterable[UUID]
+) -> dict[UUID, JobMetricsPoint]:
+    subquery = select(
+        JobMetricsPoint,
+        func.row_number()
+        .over(
+            partition_by=JobMetricsPoint.job_id,
+            order_by=JobMetricsPoint.timestamp_micro.desc(),
+        )
+        .label("row_number"),
+    ).subquery()
+    res = await session.execute(
+        select(aliased(JobMetricsPoint, subquery)).where(
+            subquery.c.row_number == 1,
+            subquery.c.job_id.in_(job_ids),
+        )
+    )
+    return {p.job_id: p for p in res.scalars().all()}
+
+
+async def _get_job_prometheus_metrics(
+    session: AsyncSession, job_ids: Iterable[UUID]
+) -> dict[UUID, JobPrometheusMetrics]:
+    res = await session.execute(
+        select(JobPrometheusMetrics).where(JobPrometheusMetrics.job_id.in_(job_ids))
+    )
+    return {p.job_id: p for p in res.scalars().all()}
+
+
+def _render_metrics(metrics: Iterable[Metric]) -> Generator[str, None, None]:
+    for metric in metrics:
+        if not metric.samples:
+            continue
+        yield f"# HELP {metric.name} {metric.documentation}"
+        yield f"# TYPE {metric.name} {metric.type}"
+        for sample in metric.samples:
+            parts: list[str] = [f"{sample.name}{{"]
+            parts.extend(",".join(f'{name}="{value}"' for name, value in sample.labels.items()))
+            parts.append(f"}} {float(sample.value)}")
+            # text_string_to_metric_families converts milliseconds to float seconds
+            if isinstance(sample.timestamp, float):
+                parts.append(f" {int(sample.timestamp * 1000)}")
+            yield "".join(parts)

dstack/_internal/server/services/proxy/__init__.py

@@ -0,0 +1,3 @@
+"""
+Dependencies for dstack-proxy that allow it to run as part of dstack-server.
+"""

dstack/_internal/server/services/proxy/auth.py

@@ -0,0 +1,12 @@
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from dstack._internal.proxy.lib.auth import BaseProxyAuthProvider
+from dstack._internal.server.security.permissions import is_project_member
+
+
+class ServerProxyAuthProvider(BaseProxyAuthProvider):
+    def __init__(self, session: AsyncSession) -> None:
+        self.session = session
+
+    async def is_project_member(self, project_name: str, token: str) -> bool:
+        return await is_project_member(self.session, project_name, token)

dstack/_internal/server/services/proxy/deps.py

@@ -0,0 +1,18 @@
+from typing import AsyncGenerator
+
+from dstack._internal.proxy.lib.auth import BaseProxyAuthProvider
+from dstack._internal.proxy.lib.deps import ProxyDependencyInjector
+from dstack._internal.proxy.lib.repo import BaseProxyRepo
+from dstack._internal.server.db import get_session_ctx
+from dstack._internal.server.services.proxy.auth import ServerProxyAuthProvider
+from dstack._internal.server.services.proxy.repo import ServerProxyRepo
+
+
+class ServerProxyDependencyInjector(ProxyDependencyInjector):
+    async def get_repo(self) -> AsyncGenerator[BaseProxyRepo, None]:
+        async with get_session_ctx() as session:
+            yield ServerProxyRepo(session)
+
+    async def get_auth_provider(self) -> AsyncGenerator[BaseProxyAuthProvider, None]:
+        async with get_session_ctx() as session:
+            yield ServerProxyAuthProvider(session)

dstack/_internal/server/services/proxy/repo.py

@@ -0,0 +1,189 @@
+from typing import List, Optional
+
+import pydantic
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import joinedload
+
+import dstack._internal.server.services.jobs as jobs_services
+from dstack._internal.core.consts import DSTACK_RUNNER_SSH_PORT
+from dstack._internal.core.models.backends.base import BackendType
+from dstack._internal.core.models.configurations import ServiceConfiguration
+from dstack._internal.core.models.instances import RemoteConnectionInfo, SSHConnectionParams
+from dstack._internal.core.models.runs import (
+    JobProvisioningData,
+    JobSpec,
+    JobStatus,
+    RunSpec,
+    RunStatus,
+    ServiceSpec,
+    get_service_port,
+)
+from dstack._internal.core.models.services import AnyModel
+from dstack._internal.proxy.lib.models import (
+    AnyModelFormat,
+    ChatModel,
+    OpenAIChatModelFormat,
+    Project,
+    Replica,
+    Service,
+    TGIChatModelFormat,
+)
+from dstack._internal.proxy.lib.repo import BaseProxyRepo
+from dstack._internal.server.models import JobModel, ProjectModel, RunModel
+from dstack._internal.server.settings import DEFAULT_SERVICE_CLIENT_MAX_BODY_SIZE
+from dstack._internal.utils.common import get_or_error
+
+
+class ServerProxyRepo(BaseProxyRepo):
+    """
+    A repo implementation used by dstack-proxy running within dstack-server.
+    Retrieves data from dstack-server's database.
+    """
+
+    def __init__(self, session: AsyncSession) -> None:
+        self.session = session
+
+    async def get_service(self, project_name: str, run_name: str) -> Optional[Service]:
+        res = await self.session.execute(
+            select(JobModel)
+            .join(JobModel.project)
+            .join(JobModel.run)
+            .where(
+                ProjectModel.name == project_name,
+                RunModel.gateway_id.is_(None),
+                JobModel.run_name == run_name,
+                JobModel.status == JobStatus.RUNNING,
+                JobModel.registered == True,
+                JobModel.job_num == 0,
+            )
+            .options(
+                joinedload(JobModel.run),
+                joinedload(JobModel.instance),
+            )
+        )
+        jobs = res.unique().scalars().all()
+        if not len(jobs):
+            return None
+        run = jobs[0].run
+        run_spec = RunSpec.__response__.parse_raw(run.run_spec)
+        if not isinstance(run_spec.configuration, ServiceConfiguration):
+            return None
+        replicas = []
+        for job in jobs:
+            jpd: JobProvisioningData = JobProvisioningData.__response__.parse_raw(
+                job.job_provisioning_data
+            )
+            assert jpd.hostname is not None
+            assert jpd.ssh_port is not None
+            if not jpd.dockerized:
+                ssh_destination = f"{jpd.username}@{jpd.hostname}"
+                ssh_port = jpd.ssh_port
+                ssh_proxy = jpd.ssh_proxy
+            else:
+                ssh_destination = "root@localhost"
+                ssh_port = DSTACK_RUNNER_SSH_PORT
+                job_submission = jobs_services.job_model_to_job_submission(job)
+                jrd = job_submission.job_runtime_data
+                if jrd is not None and jrd.ports is not None:
+                    ssh_port = jrd.ports.get(ssh_port, ssh_port)
+                ssh_proxy = SSHConnectionParams(
+                    hostname=jpd.hostname,
+                    username=jpd.username,
+                    port=jpd.ssh_port,
+                )
+                if jpd.backend == BackendType.LOCAL:
+                    ssh_proxy = None
+            ssh_head_proxy: Optional[SSHConnectionParams] = None
+            ssh_head_proxy_private_key: Optional[str] = None
+            instance = get_or_error(job.instance)
+            if instance.remote_connection_info is not None:
+                rci = RemoteConnectionInfo.__response__.parse_raw(instance.remote_connection_info)
+                if rci.ssh_proxy is not None:
+                    ssh_head_proxy = rci.ssh_proxy
+                    ssh_head_proxy_private_key = get_or_error(rci.ssh_proxy_keys)[0].private
+            job_spec: JobSpec = JobSpec.__response__.parse_raw(job.job_spec_data)
+            replica = Replica(
+                id=job.id.hex,
+                app_port=get_service_port(job_spec, run_spec.configuration),
+                ssh_destination=ssh_destination,
+                ssh_port=ssh_port,
+                ssh_proxy=ssh_proxy,
+                ssh_head_proxy=ssh_head_proxy,
+                ssh_head_proxy_private_key=ssh_head_proxy_private_key,
+            )
+            replicas.append(replica)
+        return Service(
+            project_name=project_name,
+            run_name=run.run_name,
+            domain=None,
+            https=None,
+            auth=run_spec.configuration.auth,
+            client_max_body_size=DEFAULT_SERVICE_CLIENT_MAX_BODY_SIZE,
+            strip_prefix=run_spec.configuration.strip_prefix,
+            replicas=tuple(replicas),
+        )
+
+    async def list_models(self, project_name: str) -> List[ChatModel]:
+        res = await self.session.execute(
+            select(RunModel)
+            .join(RunModel.project)
+            .where(
+                ProjectModel.name == project_name,
+                RunModel.gateway_id.is_(None),
+                RunModel.service_spec.is_not(None),
+                RunModel.status == RunStatus.RUNNING,
+            )
+        )
+        models = []
+        for run in res.scalars().all():
+            service_spec: ServiceSpec = ServiceSpec.__response__.parse_raw(run.service_spec)
+            model_spec = service_spec.model
+            model_options_obj = service_spec.options.get("openai", {}).get("model")
+            if model_spec is None or model_options_obj is None:
+                continue
+            model_options = pydantic.parse_obj_as(AnyModel, model_options_obj)  # type: ignore[arg-type]
+            model = ChatModel(
+                project_name=project_name,
+                name=model_spec.name,
+                created_at=run.submitted_at,
+                run_name=run.run_name,
+                format_spec=_model_options_to_format_spec(model_options),
+            )
+            models.append(model)
+        return models
+
+    async def get_model(self, project_name: str, name: str) -> Optional[ChatModel]:
+        models = await self.list_models(project_name)
+        models = [m for m in models if m.name == name]
+        if not models:
+            return None
+        # If there are many models with the same name, choose the most recent
+        return max(models, key=lambda m: m.created_at)
+
+    async def get_project(self, name: str) -> Optional[Project]:
+        res = await self.session.execute(select(ProjectModel).where(ProjectModel.name == name))
+        project = res.scalar_one_or_none()
+        if project is None:
+            return None
+        return Project(
+            name=project.name,
+            ssh_private_key=project.ssh_private_key,
+        )
+
+
+def _model_options_to_format_spec(model: AnyModel) -> AnyModelFormat:
+    if model.type == "chat":
+        if model.format == "openai":
+            return OpenAIChatModelFormat(prefix=model.prefix)
+        elif model.format == "tgi":
+            assert model.chat_template is not None
+            assert model.eos_token is not None
+            return TGIChatModelFormat(
+                chat_template=model.chat_template,
+                eos_token=model.eos_token,
+            )
+        else:
+            raise RuntimeError(f"Unexpected model format {model.format}")
+    else:
+        raise RuntimeError(f"Unexpected model type {model.type}")

dstack/_internal/server/services/proxy/routers/service_proxy.py

@@ -0,0 +1,49 @@
+from fastapi import APIRouter, Depends, Request, status
+from fastapi.datastructures import URL
+from fastapi.responses import RedirectResponse, Response
+from typing_extensions import Annotated
+
+from dstack._internal.proxy.lib.deps import (
+    ProxyAuth,
+    ProxyAuthContext,
+    get_proxy_repo,
+    get_service_connection_pool,
+)
+from dstack._internal.proxy.lib.repo import BaseProxyRepo
+from dstack._internal.proxy.lib.services.service_connection import ServiceConnectionPool
+from dstack._internal.server.services.proxy.services import service_proxy
+
+router = APIRouter()
+
+
+@router.get("/{project_name}/{run_name}")
+@router.post("/{project_name}/{run_name}")
+@router.put("/{project_name}/{run_name}")
+@router.delete("/{project_name}/{run_name}")
+@router.patch("/{project_name}/{run_name}")
+@router.head("/{project_name}/{run_name}")
+async def redirect_to_service_root(request: Request, project_name: str, run_name: str) -> Response:
+    url = URL(str(request.url))
+    url = url.replace(path=url.path + "/")
+    return RedirectResponse(url, status.HTTP_308_PERMANENT_REDIRECT)
+
+
+@router.get("/{project_name}/{run_name}/{path:path}")
+@router.post("/{project_name}/{run_name}/{path:path}")
+@router.put("/{project_name}/{run_name}/{path:path}")
+@router.delete("/{project_name}/{run_name}/{path:path}")
+@router.patch("/{project_name}/{run_name}/{path:path}")
+@router.head("/{project_name}/{run_name}/{path:path}")
+@router.options("/{project_name}/{run_name}/{path:path}")
+async def service_reverse_proxy(
+    project_name: str,
+    run_name: str,
+    path: str,
+    request: Request,
+    auth: Annotated[ProxyAuthContext, Depends(ProxyAuth(auto_enforce=False))],
+    repo: Annotated[BaseProxyRepo, Depends(get_proxy_repo)],
+    service_conn_pool: Annotated[ServiceConnectionPool, Depends(get_service_connection_pool)],
+) -> Response:
+    return await service_proxy.proxy(
+        project_name, run_name, path, request, auth, repo, service_conn_pool
+    )