dstack 0.0.9__py3-none-any.whl → 0.20.7__py3-none-any.whl

dstack/_internal/proxy/gateway/main.py

@@ -0,0 +1,17 @@
+import logging
+
+from dstack._internal.proxy.gateway.app import make_app
+
+
+def configure_logging(level: int = logging.INFO):
+    formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+    handler = logging.StreamHandler()
+    handler.setFormatter(formatter)
+
+    logger = logging.getLogger("dstack")
+    logger.setLevel(level)
+    logger.addHandler(handler)
+
+
+configure_logging(logging.DEBUG)
+app = make_app()

dstack/_internal/proxy/gateway/models.py

@@ -0,0 +1,23 @@
+"""Things stored in GatewayProxyRepo in addition to those from BaseProxyRepo."""
+
+from typing import Optional
+
+from pydantic import AnyHttpUrl
+
+from dstack._internal.proxy.lib.models import ImmutableModel
+
+
+class ModelEntrypoint(ImmutableModel):
+    project_name: str
+    domain: str
+    https: bool
+
+
+class ACMESettings(ImmutableModel):
+    server: Optional[AnyHttpUrl] = None
+    eab_kid: Optional[str] = None
+    eab_hmac_key: Optional[str] = None
+
+
+class GlobalProxyConfig(ImmutableModel):
+    acme_settings: ACMESettings = ACMESettings()

dstack/_internal/proxy/gateway/repo/repo.py

@@ -0,0 +1,121 @@
+from contextlib import asynccontextmanager
+from itertools import chain
+from pathlib import Path
+from typing import Optional
+
+from aiorwlock import RWLock
+from pydantic import BaseModel
+
+from dstack._internal.proxy.gateway.models import GlobalProxyConfig, ModelEntrypoint
+from dstack._internal.proxy.lib.models import ChatModel, Project, Service
+from dstack._internal.proxy.lib.repo import BaseProxyRepo
+from dstack._internal.utils.common import run_async
+
+
+class State(BaseModel):
+    services: dict[str, dict[str, Service]] = {}
+    models: dict[str, dict[str, ChatModel]] = {}
+    entrypoints: dict[str, ModelEntrypoint] = {}
+    projects: dict[str, Project] = {}
+    config: GlobalProxyConfig = GlobalProxyConfig()
+
+
+class GatewayProxyRepo(BaseProxyRepo):
+    """
+    Repo implementation used on gateways. Stores state in memory and maintains a copy on disk.
+    """
+
+    def __init__(self, state: Optional[State] = None, file: Optional[Path] = None) -> None:
+        self._state = state or State()
+        self._file = file
+        self._lock = RWLock()
+
+    async def list_services(self) -> list[Service]:
+        async with self.reader():
+            services_by_project = (
+                project_services.values() for project_services in self._state.services.values()
+            )
+            return list(chain(*services_by_project))
+
+    async def get_service(self, project_name: str, run_name: str) -> Optional[Service]:
+        async with self.reader():
+            return self._state.services.get(project_name, {}).get(run_name)
+
+    async def set_service(self, service: Service) -> None:
+        async with self.writer():
+            self._state.services.setdefault(service.project_name, {})[service.run_name] = service
+
+    async def delete_service(self, project_name: str, run_name: str) -> None:
+        async with self.writer():
+            project_services = self._state.services.get(project_name, {})
+            project_services.pop(run_name, None)
+            if not project_services:
+                self._state.services.pop(project_name, None)
+
+    async def list_models(self, project_name: str) -> list[ChatModel]:
+        async with self.reader():
+            return list(self._state.models.get(project_name, {}).values())
+
+    async def get_model(self, project_name: str, name: str) -> Optional[ChatModel]:
+        async with self.reader():
+            return self._state.models.get(project_name, {}).get(name)
+
+    async def set_model(self, model: ChatModel) -> None:
+        async with self.writer():
+            self._state.models.setdefault(model.project_name, {})[model.name] = model
+
+    async def delete_models_by_run(self, project_name: str, run_name: str) -> None:
+        async with self.writer():
+            project_models = self._state.models.get(project_name, {})
+            models_to_delete = [m for m in project_models.values() if m.run_name == run_name]
+            for model in models_to_delete:
+                project_models.pop(model.name, None)
+            if not project_models:
+                self._state.models.pop(project_name, None)
+
+    async def list_entrypoints(self) -> list[ModelEntrypoint]:
+        async with self.reader():
+            return list(self._state.entrypoints.values())
+
+    async def set_entrypoint(self, entrypoint: ModelEntrypoint) -> None:
+        async with self.writer():
+            self._state.entrypoints[entrypoint.project_name] = entrypoint
+
+    async def get_project(self, name: str) -> Optional[Project]:
+        async with self.reader():
+            return self._state.projects.get(name)
+
+    async def set_project(self, project: Project) -> None:
+        async with self.writer():
+            self._state.projects[project.name] = project
+
+    async def get_config(self) -> GlobalProxyConfig:
+        async with self.reader():
+            return self._state.config
+
+    async def set_config(self, config: GlobalProxyConfig) -> None:
+        async with self.writer():
+            self._state.config = config
+
+    @asynccontextmanager
+    async def reader(self):
+        async with self._lock.reader:
+            yield
+
+    @asynccontextmanager
+    async def writer(self):
+        async with self._lock.writer:
+            yield
+            await run_async(self.save)
+
+    @staticmethod
+    def load(state_file: Path) -> "GatewayProxyRepo":
+        if state_file.exists():
+            state = State.parse_file(state_file)
+        else:
+            state = None
+        return GatewayProxyRepo(state=state, file=state_file)
+
+    def save(self) -> None:
+        if self._file is not None:
+            self._file.write_text(self._state.json())

dstack/_internal/proxy/gateway/repo/state_v1.py

@@ -0,0 +1,164 @@
+"""
+Migration from the legacy state.json file of dstack-gateway to the new
+state-v2.json file of dstack-proxy.
+"""
+
+import json
+from datetime import datetime
+from pathlib import Path
+from typing import Iterable, Optional
+
+from dstack._internal.core.models.instances import SSHConnectionParams
+from dstack._internal.proxy.gateway.models import ACMESettings, GlobalProxyConfig, ModelEntrypoint
+from dstack._internal.proxy.gateway.repo.repo import GatewayProxyRepo, State
+from dstack._internal.proxy.lib.models import (
+    AnyModelFormat,
+    ChatModel,
+    OpenAIChatModelFormat,
+    Project,
+    Replica,
+    Service,
+    TGIChatModelFormat,
+)
+from dstack._internal.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+def migrate_from_state_v1(v1_file: Path, v2_file: Path, keys_dir: Path) -> None:
+    if v2_file.exists() or not v1_file.exists():
+        return
+    state_v1 = json.loads(v1_file.read_text())
+    state = parse_state_v1(state_v1, keys_dir)
+    repo = GatewayProxyRepo(state=state, file=v2_file)
+    repo.save()
+    logger.info("Successfully migrated state from %s to %s", v1_file, v2_file)
+
+
+def parse_state_v1(state_v1: dict, keys_dir: Path) -> State:
+    services, models = get_services_models(state_v1)
+    return State(
+        services=services,
+        models=models,
+        entrypoints=get_entrypoints(state_v1.get("store", {})),
+        projects=get_projects(state_v1.get("store", {}).get("projects", {}), keys_dir),
+        config=get_config(state_v1.get("store", {}).get("nginx", {}).get("acme_settings", {})),
+    )
+
+
+def get_services_models(
+    state_v1: dict,
+) -> tuple[dict[str, dict[str, Service]], dict[str, dict[str, ChatModel]]]:
+    service_id_to_project_name = {}
+    for project_name, project_services in state_v1.get("store", {}).get("projects", {}).items():
+        for service_id in project_services:
+            service_id_to_project_name[service_id] = project_name
+
+    services: dict[str, dict[str, Service]] = {}
+    models: dict[str, dict[str, ChatModel]] = {}
+    for service in state_v1.get("store", {}).get("services", {}).values():
+        project_name = service_id_to_project_name[service["id"]]
+        replicas = []
+        for replica in service.get("replicas", []):
+            replicas.append(parse_replica(replica))
+        parsed_service = Service(
+            project_name=project_name,
+            run_name=service["domain"].split(".")[0],
+            domain=service["domain"],
+            https=service.get("https", True),
+            auth=service["auth"],
+            client_max_body_size=service.get("client_max_body_size", 1024 * 1024),
+            replicas=tuple(replicas),
+        )
+        services.setdefault(project_name, {})[parsed_service.run_name] = parsed_service
+        if model := service.get("options", {}).get("openai", {}).get("model", {}):
+            parsed_model = parse_model(
+                project_name, parsed_service.run_name, model, state_v1["openai"]["index"]
+            )
+            if parsed_model is not None:
+                models.setdefault(project_name, {})[parsed_model.name] = parsed_model
+
+    return services, models
+
+
+def parse_replica(replica: dict) -> Replica:
+    ssh_proxy = None
+    if (ssh_proxy_destination := replica.get("ssh_jump_host")) and (
+        ssh_proxy_port := replica.get("ssh_jump_port")
+    ):
+        proxy_user, proxy_host = ssh_proxy_destination.split("@")
+        ssh_proxy = SSHConnectionParams(
+            hostname=proxy_host,
+            username=proxy_user,
+            port=ssh_proxy_port,
+        )
+    return Replica(
+        id=replica["id"],
+        app_port=replica["app_port"],
+        ssh_destination=replica["ssh_host"],
+        ssh_port=replica["ssh_port"],
+        ssh_proxy=ssh_proxy,
+    )
+
+
+def parse_model(
+    project_name: str, run_name: str, model: dict, openai_index: dict
+) -> Optional[ChatModel]:
+    created_ts = (
+        openai_index.get(project_name, {}).get("chat", {}).get(model["name"], {}).get("created")
+    )
+    if created_ts is None:
+        # some models can be missing in the index, most likely due to a bug
+        return None
+    format_spec: AnyModelFormat
+    if model["format"] == "tgi":
+        format_spec = TGIChatModelFormat(
+            chat_template=model["chat_template"], eos_token=model["eos_token"]
+        )
+    else:
+        format_spec = OpenAIChatModelFormat(prefix=model["prefix"])
+    return ChatModel(
+        project_name=project_name,
+        name=model["name"],
+        created_at=datetime.fromtimestamp(created_ts),
+        run_name=run_name,
+        format_spec=format_spec,
+    )
+
+
+def get_entrypoints(store: dict) -> dict[str, ModelEntrypoint]:
+    entrypoint_domain_to_project_name = {}
+    for entrypoint_domain, (project_name, _) in store.get("entrypoints", {}).items():
+        entrypoint_domain_to_project_name[entrypoint_domain] = project_name
+
+    entrypoints = {}
+    for site_config in store.get("nginx", {}).get("configs", {}).values():
+        if site_config["type"] == "entrypoint":
+            entrypoint = ModelEntrypoint(
+                project_name=entrypoint_domain_to_project_name[site_config["domain"]],
+                domain=site_config["domain"],
+                https=site_config["https"],
+            )
+            entrypoints[entrypoint.project_name] = entrypoint
+
+    return entrypoints
+
+
+def get_projects(project_names: Iterable[str], keys_dir: Path) -> dict[str, Project]:
+    projects = {}
+    for project_name in project_names:
+        projects[project_name] = Project(
+            name=project_name,
+            ssh_private_key=(keys_dir / project_name).read_text(),
+        )
+    return projects
+
+
+def get_config(acme_settings: dict) -> GlobalProxyConfig:
+    return GlobalProxyConfig(
+        acme_settings=ACMESettings(
+            server=acme_settings.get("server"),
+            eab_kid=acme_settings.get("eab_kid"),
+            eab_hmac_key=acme_settings.get("eab_hmac_key"),
+        )
+    )

dstack/_internal/proxy/gateway/resources/nginx/00-log-format.conf

@@ -0,0 +1,11 @@
+log_format dstack_stat '$time_iso8601 $host $status $request_time $dstack_replica_hit';
+
+
+# A hack to avoid this Nginx reload error when no services are registered:
+#     nginx: [emerg] unknown "dstack_replica_hit" variable
+server {
+    listen unix:/tmp/dstack-dummy-nginx.sock;
+    server_name placeholder.local;
+    deny all;
+    set $dstack_replica_hit 0;
+}

dstack/_internal/proxy/gateway/resources/nginx/entrypoint.jinja2

@@ -0,0 +1,27 @@
+server {
+    server_name {{ domain }};
+    location / {
+        proxy_pass http://localhost:{{ proxy_port }}/api/models/{{ project_name }}/;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header Host $host;
+        proxy_read_timeout 300s;
+    }
+    listen 80;
+    {% if https %}
+    listen 443 ssl;
+    ssl_certificate /etc/letsencrypt/live/{{ domain }}/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/{{ domain }}/privkey.pem;
+    include /etc/letsencrypt/options-ssl-nginx.conf;
+    ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
+    set $force_https 1;
+    if ($scheme = "https") {
+        set $force_https 0;
+    }
+    if ($remote_addr = 127.0.0.1) {
+        set $force_https 0;
+    }
+    if ($force_https) {
+        return 301 https://$host$request_uri;
+    }
+    {% endif %}
+}

dstack/_internal/proxy/gateway/resources/nginx/router_workers.jinja2

@@ -0,0 +1,23 @@
+{% for replica in replicas %}
+# Worker {{ loop.index }}
+upstream router_worker_{{ domain|replace('.', '_') }}_{{ ports[loop.index0] }}_upstream {
+    server unix:{{ replica.socket }};
+}
+
+server {
+    listen 127.0.0.1:{{ ports[loop.index0] }};
+    access_log off; # disable access logs for this internal endpoint
+
+    proxy_read_timeout 300s;
+    proxy_send_timeout 300s;
+
+    location / {
+        proxy_pass http://router_worker_{{ domain|replace('.', '_') }}_{{ ports[loop.index0] }}_upstream;
+        proxy_http_version 1.1;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header Connection "";
+        proxy_set_header Upgrade $http_upgrade;
+    }
+}
+{% endfor %}

dstack/_internal/proxy/gateway/resources/nginx/service.jinja2

@@ -0,0 +1,105 @@
+{% for zone in limit_req_zones %}
+limit_req_zone {{ zone.key }} zone={{ zone.name }}:10m rate={{ zone.rpm }}r/m;
+{% endfor %}
+
+{% if replicas %}
+upstream {{ domain }}.upstream {
+    {% if router_port is not none %}
+    server 127.0.0.1:{{ router_port }};  # SGLang router on the gateway
+    {% else %}
+    {% for replica in replicas %}
+    server unix:{{ replica.socket }};  # replica {{ replica.id }}
+    {% endfor %}
+    {% endif %}
+}
+{% else %}
+
+{% endif %}
+server {
+    server_name {{ domain }};
+    limit_req_status 429;
+    set $dstack_replica_hit 0;
+    access_log {{ access_log_path }} dstack_stat;
+    client_max_body_size {{ client_max_body_size }};
+
+    {% for location in locations %}
+    location {{ location.prefix }} {
+        {% if auth %}
+        auth_request /_dstack_auth;
+        {% endif %}
+
+        try_files /nonexistent @$http_upgrade;
+
+        {% if location.limit_req %}
+        limit_req zone={{ location.limit_req.zone }}{% if location.limit_req.burst %} burst={{ location.limit_req.burst }} nodelay{% endif %};
+        {% endif %}
+    }
+    {% endfor %}
+
+    {# For SGLang router: block all requests except whitelisted locations added dynamically above #}
+    {% if router is not none and router.type == "sglang" %}
+    location / {
+        return 403;
+    }
+    {% endif %}
+
+    location @websocket {
+        set $dstack_replica_hit 1;
+        {% if replicas %}
+        proxy_pass http://{{ domain }}.upstream;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header Host $host;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "Upgrade";
+        proxy_read_timeout 300s;
+        {% else %}
+        return 503;
+        {% endif %}
+    }
+    location @ {
+        set $dstack_replica_hit 1;
+        {% if replicas %}
+        proxy_pass http://{{ domain }}.upstream;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header Host $host;
+        proxy_read_timeout 300s;
+        {% else %}
+        return 503;
+        {% endif %}
+    }
+
+    {% if auth %}
+    location = /_dstack_auth {
+        internal;
+        if ($remote_addr = 127.0.0.1) {
+            # for requests from the gateway app, e.g. from the OpenAI-compatible API
+            return 200;
+        }
+        proxy_pass http://localhost:{{ proxy_port }}/api/auth/{{ project_name }};
+        proxy_pass_request_body off;
+        proxy_set_header Content-Length "";
+        proxy_set_header X-Original-URI $request_uri;
+        proxy_set_header Authorization $http_authorization;
+    }
+    {% endif %}
+
+    listen 80;
+    {% if https %}
+    listen 443 ssl;
+    ssl_certificate /etc/letsencrypt/live/{{ domain }}/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/{{ domain }}/privkey.pem;
+    include /etc/letsencrypt/options-ssl-nginx.conf;
+    ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
+    set $force_https 1;
+    if ($scheme = "https") {
+        set $force_https 0;
+    }
+    if ($remote_addr = 127.0.0.1) {
+        set $force_https 0;
+    }
+    if ($force_https) {
+        return 301 https://$host$request_uri;
+    }
+    {% endif %}
+}

dstack/_internal/proxy/gateway/routers/auth.py

@@ -0,0 +1,10 @@
+from fastapi import APIRouter, Depends
+
+from dstack._internal.proxy.lib.deps import ProxyAuth
+
+router = APIRouter()
+
+
+@router.get("/{project_name}", dependencies=[Depends(ProxyAuth(auto_enforce=True))])
+async def get_auth():
+    return {"status": "ok"}

dstack/_internal/proxy/gateway/routers/config.py

@@ -0,0 +1,28 @@
+from typing import Annotated
+
+from fastapi import APIRouter, Depends
+
+from dstack._internal.proxy.gateway.deps import get_gateway_proxy_repo
+from dstack._internal.proxy.gateway.models import ACMESettings, GlobalProxyConfig
+from dstack._internal.proxy.gateway.repo.repo import GatewayProxyRepo
+from dstack._internal.proxy.gateway.schemas.common import OkResponse
+from dstack._internal.proxy.gateway.schemas.config import ConfigRequest
+
+router = APIRouter()
+
+
+@router.post("")
+async def update_global_config(
+    body: ConfigRequest,
+    repo: Annotated[GatewayProxyRepo, Depends(get_gateway_proxy_repo)],
+) -> OkResponse:
+    await repo.set_config(
+        GlobalProxyConfig(
+            acme_settings=ACMESettings(
+                server=body.acme_server,
+                eab_kid=body.acme_eab_kid,
+                eab_hmac_key=body.acme_eab_hmac_key,
+            ),
+        ),
+    )
+    return OkResponse()

dstack/_internal/proxy/gateway/routers/registry.py

@@ -0,0 +1,124 @@
+from fastapi import APIRouter, Depends
+from typing_extensions import Annotated
+
+import dstack._internal.proxy.gateway.services.registry as registry_services
+from dstack._internal.proxy.gateway.deps import get_gateway_proxy_repo, get_nginx
+from dstack._internal.proxy.gateway.repo.repo import GatewayProxyRepo
+from dstack._internal.proxy.gateway.schemas.common import OkResponse
+from dstack._internal.proxy.gateway.schemas.registry import (
+    RegisterEntrypointRequest,
+    RegisterReplicaRequest,
+    RegisterServiceRequest,
+)
+from dstack._internal.proxy.gateway.services.nginx import Nginx
+from dstack._internal.proxy.lib.deps import get_service_connection_pool
+from dstack._internal.proxy.lib.services.service_connection import ServiceConnectionPool
+
+router = APIRouter(prefix="/{project_name}")
+
+
+@router.post("/services/register")
+async def register_service(
+    project_name: str,
+    body: RegisterServiceRequest,
+    repo: Annotated[GatewayProxyRepo, Depends(get_gateway_proxy_repo)],
+    nginx: Annotated[Nginx, Depends(get_nginx)],
+    service_conn_pool: Annotated[ServiceConnectionPool, Depends(get_service_connection_pool)],
+) -> OkResponse:
+    await registry_services.register_service(
+        project_name=project_name.lower(),
+        run_name=body.run_name.lower(),
+        domain=body.domain.lower(),
+        https=body.https,
+        rate_limits=body.rate_limits,
+        auth=body.auth,
+        client_max_body_size=body.client_max_body_size,
+        model=body.options.openai.model if body.options.openai is not None else None,
+        ssh_private_key=body.ssh_private_key,
+        repo=repo,
+        router=body.router,
+        nginx=nginx,
+        service_conn_pool=service_conn_pool,
+    )
+    return OkResponse()
+
+
+@router.post("/services/{run_name}/unregister")
+async def unregister_service(
+    project_name: str,
+    run_name: str,
+    repo: Annotated[GatewayProxyRepo, Depends(get_gateway_proxy_repo)],
+    nginx: Annotated[Nginx, Depends(get_nginx)],
+    service_conn_pool: Annotated[ServiceConnectionPool, Depends(get_service_connection_pool)],
+) -> OkResponse:
+    await registry_services.unregister_service(
+        project_name=project_name.lower(),
+        run_name=run_name.lower(),
+        repo=repo,
+        nginx=nginx,
+        service_conn_pool=service_conn_pool,
+    )
+    return OkResponse()
+
+
+@router.post("/services/{run_name}/replicas/register")
+async def register_replica(
+    project_name: str,
+    run_name: str,
+    body: RegisterReplicaRequest,
+    repo: Annotated[GatewayProxyRepo, Depends(get_gateway_proxy_repo)],
+    nginx: Annotated[Nginx, Depends(get_nginx)],
+    service_conn_pool: Annotated[ServiceConnectionPool, Depends(get_service_connection_pool)],
+) -> OkResponse:
+    await registry_services.register_replica(
+        project_name=project_name.lower(),
+        run_name=run_name.lower(),
+        replica_id=body.job_id,
+        app_port=body.app_port,
+        ssh_destination=body.ssh_host,
+        ssh_port=body.ssh_port,
+        ssh_proxy=body.ssh_proxy,
+        ssh_head_proxy=body.ssh_head_proxy,
+        ssh_head_proxy_private_key=body.ssh_head_proxy_private_key,
+        repo=repo,
+        nginx=nginx,
+        service_conn_pool=service_conn_pool,
+    )
+    return OkResponse()
+
+
+@router.post("/services/{run_name}/replicas/{job_id}/unregister")
+async def unregister_replica(
+    project_name: str,
+    run_name: str,
+    job_id: str,
+    repo: Annotated[GatewayProxyRepo, Depends(get_gateway_proxy_repo)],
+    nginx: Annotated[Nginx, Depends(get_nginx)],
+    service_conn_pool: Annotated[ServiceConnectionPool, Depends(get_service_connection_pool)],
+) -> OkResponse:
+    await registry_services.unregister_replica(
+        project_name=project_name.lower(),
+        run_name=run_name.lower(),
+        replica_id=job_id,
+        repo=repo,
+        nginx=nginx,
+        service_conn_pool=service_conn_pool,
+    )
+    return OkResponse()
+
+
+@router.post("/entrypoints/register")
+async def register_entrypoint(
+    project_name: str,
+    body: RegisterEntrypointRequest,
+    repo: Annotated[GatewayProxyRepo, Depends(get_gateway_proxy_repo)],
+    nginx: Annotated[Nginx, Depends(get_nginx)],
+) -> OkResponse:
+    await registry_services.register_model_entrypoint(
+        project_name=project_name.lower(),
+        domain=body.domain.lower(),
+        https=body.https,
+        repo=repo,
+        nginx=nginx,
+    )
+    return OkResponse()

dstack/_internal/proxy/gateway/routers/stats.py

@@ -0,0 +1,18 @@
+from typing import Annotated
+
+from fastapi import APIRouter, Depends
+
+from dstack._internal.proxy.gateway.deps import get_gateway_proxy_repo, get_stats_collector
+from dstack._internal.proxy.gateway.repo.repo import GatewayProxyRepo
+from dstack._internal.proxy.gateway.schemas.stats import ServiceStats
+from dstack._internal.proxy.gateway.services.stats import StatsCollector, get_service_stats
+
+router = APIRouter()
+
+
+@router.get("/collect")
+async def collect_stats(
+    repo: Annotated[GatewayProxyRepo, Depends(get_gateway_proxy_repo)],
+    collector: Annotated[StatsCollector, Depends(get_stats_collector)],
+) -> list[ServiceStats]:
+    return await get_service_stats(repo, collector)