latencyops 0.1.0__py3-none-any.whl

latencyops/__init__.py

@@ -0,0 +1,33 @@
+"""LatencyOps: latency measurement and quality-aware inference planning."""
+
+from .capabilities import EnforcedPlan, ProviderCapabilities, enforce_plan
+from .controller import ProactiveController
+from .gateway import GatewayService, create_server
+from .models import LatencyContract, RequestProfile, StreamChunk, StreamingResult
+from .policy import AdaptiveProviderSelector, ElasticInferencePlanner
+from .orchestrators import PrometheusMetricsCollector, SGLangMetricsCollector, TensorRTLLMMetricsCollector, VLLMMetricsCollector
+from .signals import RuntimeSignalNormalizer
+from .scheduling import CachePressureSignal, InMemoryRequestQueue, PriorityScheduler
+
+__all__ = [
+    "AdaptiveProviderSelector",
+    "CachePressureSignal",
+    "ElasticInferencePlanner",
+    "EnforcedPlan",
+    "GatewayService",
+    "InMemoryRequestQueue",
+    "LatencyContract",
+    "PriorityScheduler",
+    "PrometheusMetricsCollector",
+    "ProactiveController",
+    "ProviderCapabilities",
+    "RequestProfile",
+    "RuntimeSignalNormalizer",
+    "SGLangMetricsCollector",
+    "TensorRTLLMMetricsCollector",
+    "VLLMMetricsCollector",
+    "StreamChunk",
+    "StreamingResult",
+    "create_server",
+    "enforce_plan",
+]

latencyops/adapters.py

@@ -0,0 +1,186 @@
+"""Provider adapters built on the standard library HTTP client."""
+
+import json
+from dataclasses import dataclass
+from typing import Callable, Iterable, Mapping
+from urllib.error import HTTPError, URLError
+from urllib.request import Request, urlopen
+
+from .capabilities import ProviderCapabilities
+from .models import InferencePlan, StreamChunk
+
+
+class ProviderError(RuntimeError):
+    """Normalized provider failure without including request content."""
+
+
+@dataclass(frozen=True)
+class ProviderHealth:
+    """Result of a provider health probe."""
+
+    healthy: bool
+    detail: str
+
+
+class CallableProvider:
+    """Wrap a local callable as a deterministic inference provider."""
+
+    def __init__(self, name: str, responder: Callable[[str, InferencePlan], str]):
+        self.name = name
+        self.responder = responder
+
+    def generate(self, prompt: str, plan: InferencePlan) -> str:
+        return self.responder(prompt, plan)
+
+    def stream(self, prompt: str, plan: InferencePlan) -> Iterable[StreamChunk]:
+        yield StreamChunk(self.generate(prompt, plan))
+
+
+class OpenAICompatibleProvider:
+    """Minimal adapter for JSON-compatible OpenAI-style completion endpoints."""
+
+    def __init__(
+        self,
+        name: str,
+        endpoint: str,
+        api_key: str | None = None,
+        timeout: float = 30.0,
+        model_names: Mapping[str, str] | None = None,
+        health_endpoint: str | None = None,
+    ):
+        self.name = name
+        self.endpoint = endpoint
+        self.api_key = api_key
+        self.timeout = timeout
+        self.model_names = dict(model_names or {})
+        self.health_endpoint = health_endpoint
+        self.capabilities = ProviderCapabilities()
+
+    def _model_name(self, plan: InferencePlan) -> str:
+        return self.model_names.get(plan.model_tier, plan.model_tier)
+
+    def _headers(self, streaming: bool = False) -> dict[str, str]:
+        headers = {"Content-Type": "application/json"}
+        if streaming:
+            headers["Accept"] = "text/event-stream"
+        if self.api_key:
+            headers["Authorization"] = f"Bearer {self.api_key}"
+        return headers
+
+    def generate(self, prompt: str, plan: InferencePlan) -> str:
+        payload = json.dumps({"model": self._model_name(plan), "prompt": prompt, "stream": False}).encode()
+        request = Request(self.endpoint, data=payload, headers=self._headers(), method="POST")
+        try:
+            with urlopen(request, timeout=self.timeout) as response:
+                body = json.loads(response.read().decode())
+        except (HTTPError, URLError, TimeoutError, ConnectionError, json.JSONDecodeError) as exc:
+            raise ProviderError(f"provider {self.name!r} request failed") from exc
+        choices = body.get("choices", [])
+        if not choices or "text" not in choices[0]:
+            raise ProviderError(f"provider {self.name!r} returned an invalid response")
+        return choices[0]["text"]
+
+    def stream(self, prompt: str, plan: InferencePlan) -> Iterable[StreamChunk]:
+        payload = json.dumps({"model": self._model_name(plan), "prompt": prompt, "stream": True}).encode()
+        request = Request(self.endpoint, data=payload, headers=self._headers(streaming=True), method="POST")
+        try:
+            with urlopen(request, timeout=self.timeout) as response:
+                for raw_line in response:
+                    line = raw_line.decode().strip()
+                    if not line or not line.startswith("data:"):
+                        continue
+                    data = line[5:].strip()
+                    if data == "[DONE]":
+                        break
+                    event = json.loads(data)
+                    text = event.get("choices", [{}])[0].get("text", "")
+                    if text:
+                        yield StreamChunk(text)
+        except (HTTPError, URLError, TimeoutError, ConnectionError, json.JSONDecodeError) as exc:
+            raise ProviderError(f"provider {self.name!r} stream failed") from exc
+
+    def health(self) -> ProviderHealth:
+        if not self.health_endpoint:
+            return ProviderHealth(True, "health probe not configured")
+        request = Request(self.health_endpoint, headers=self._headers(), method="GET")
+        try:
+            with urlopen(request, timeout=self.timeout) as response:
+                return ProviderHealth(200 <= response.status < 300, "HTTP health probe completed")
+        except (HTTPError, URLError, TimeoutError, ConnectionError) as exc:
+            return ProviderHealth(False, "HTTP health probe failed")
+
+
+class OpenAIChatCompatibleProvider(OpenAICompatibleProvider):
+    """Adapter for OpenAI-compatible chat-completion endpoints."""
+
+    def __init__(self, *args, chat_template_kwargs: Mapping[str, object] | None = None, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.chat_template_kwargs = dict(chat_template_kwargs or {})
+
+    def _payload(self, prompt: str, plan: InferencePlan, streaming: bool) -> dict[str, object]:
+        payload: dict[str, object] = {
+            "model": self._model_name(plan),
+            "messages": [{"role": "user", "content": prompt}],
+            "stream": streaming,
+        }
+        if self.chat_template_kwargs:
+            payload["chat_template_kwargs"] = self.chat_template_kwargs
+        return payload
+
+    def generate(self, prompt: str, plan: InferencePlan) -> str:
+        request = Request(
+            self.endpoint,
+            data=json.dumps(self._payload(prompt, plan, False)).encode(),
+            headers=self._headers(),
+            method="POST",
+        )
+        try:
+            with urlopen(request, timeout=self.timeout) as response:
+                body = json.loads(response.read().decode())
+        except (HTTPError, URLError, TimeoutError, ConnectionError, json.JSONDecodeError) as exc:
+            raise ProviderError(f"provider {self.name!r} request failed") from exc
+        choices = body.get("choices", [])
+        content = choices[0].get("message", {}).get("content") if choices else None
+        if not isinstance(content, str):
+            raise ProviderError(f"provider {self.name!r} returned an invalid chat response")
+        return content
+
+    def stream(self, prompt: str, plan: InferencePlan) -> Iterable[StreamChunk]:
+        request = Request(
+            self.endpoint,
+            data=json.dumps(self._payload(prompt, plan, True)).encode(),
+            headers=self._headers(streaming=True),
+            method="POST",
+        )
+        try:
+            with urlopen(request, timeout=self.timeout) as response:
+                for raw_line in response:
+                    line = raw_line.decode().strip()
+                    if not line or not line.startswith("data:"):
+                        continue
+                    data = line[5:].strip()
+                    if data == "[DONE]":
+                        break
+                    event = json.loads(data)
+                    choice = event.get("choices", [{}])[0]
+                    delta = choice.get("delta", {})
+                    text = delta.get("content", "")
+                    if text:
+                        yield StreamChunk(text)
+        except (HTTPError, URLError, TimeoutError, ConnectionError, json.JSONDecodeError) as exc:
+            raise ProviderError(f"provider {self.name!r} chat stream failed") from exc
+
+    def list_models(self) -> list[str]:
+        """Return model IDs visible to this provider credential."""
+        endpoint = self.endpoint.rsplit("/v1/", 1)[0] + "/v1/models"
+        request = Request(endpoint, headers=self._headers(), method="GET")
+        try:
+            with urlopen(request, timeout=self.timeout) as response:
+                body = json.loads(response.read().decode())
+        except (HTTPError, URLError, TimeoutError, ConnectionError, json.JSONDecodeError) as exc:
+            raise ProviderError(f"provider {self.name!r} model listing failed") from exc
+        return [item["id"] for item in body.get("data", []) if isinstance(item.get("id"), str)]
+
+
+class VLLMCompatibleProvider(OpenAIChatCompatibleProvider):
+    """vLLM adapter using its OpenAI-compatible chat-completions endpoint."""

latencyops/benchmark.py

@@ -0,0 +1,64 @@
+"""Small benchmark runner for provider comparisons."""
+
+from time import perf_counter
+from typing import Callable, Iterable
+
+from .models import InferencePlan, LatencySample, StreamingResult
+from .router import InferenceProvider, StreamingProvider
+
+
+class BenchmarkRunner:
+    """Run synthetic or approved prompts without retaining prompt content."""
+
+    def __init__(self, clock: Callable[[], float] = perf_counter):
+        self.clock = clock
+
+    def run(self, provider: InferenceProvider, prompts: Iterable[str], plan: InferencePlan) -> list[LatencySample]:
+        samples: list[LatencySample] = []
+        for prompt in prompts:
+            started = self.clock()
+            try:
+                provider.generate(prompt, plan)
+            except Exception:
+                elapsed_ms = (self.clock() - started) * 1000
+                samples.append(LatencySample(elapsed_ms, None, elapsed_ms, success=False))
+                continue
+            elapsed_ms = (self.clock() - started) * 1000
+            samples.append(LatencySample(elapsed_ms, None, elapsed_ms))
+        return samples
+
+    def run_streaming(
+        self, provider: StreamingProvider, prompt: str, plan: InferencePlan
+    ) -> StreamingResult:
+        """Collect one stream and measure TTFT and mean TPOT separately."""
+        started = self.clock()
+        first_chunk_at: float | None = None
+        first_chunk_token_count = 0
+        token_count = 0
+        text: list[str] = []
+        try:
+            for chunk in provider.stream(prompt, plan):
+                now = self.clock()
+                if first_chunk_at is None:
+                    first_chunk_at = now
+                    first_chunk_token_count = chunk.token_count
+                token_count += chunk.token_count
+                text.append(chunk.text)
+        except Exception:
+            elapsed_ms = (self.clock() - started) * 1000
+            return StreamingResult("".join(text), LatencySample(
+                (first_chunk_at - started) * 1000 if first_chunk_at is not None else elapsed_ms,
+                None,
+                elapsed_ms,
+                success=False,
+            ))
+
+        ended = self.clock()
+        elapsed_ms = (ended - started) * 1000
+        if first_chunk_at is None:
+            return StreamingResult("", LatencySample(elapsed_ms, None, elapsed_ms))
+        intervals = max(token_count - first_chunk_token_count, 0)
+        tpot_ms = ((ended - first_chunk_at) * 1000 / intervals) if intervals else None
+        return StreamingResult("".join(text), LatencySample(
+            (first_chunk_at - started) * 1000, tpot_ms, elapsed_ms
+        ))

latencyops/benchmark_suite.py

@@ -0,0 +1,112 @@
+"""Scenario benchmark runner for proactive policy evaluation."""
+
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+from time import sleep
+from typing import Callable
+
+from .benchmark import BenchmarkRunner
+from .models import InferencePlan, LatencyContract, LatencySample, RequestProfile
+from .router import StreamingProvider
+
+
+@dataclass(frozen=True)
+class BenchmarkScenario:
+    """A repeatable workload definition with content-free reporting metadata."""
+
+    name: str
+    prompt: str
+    contract: LatencyContract
+    profile: RequestProfile
+    cache_state: str = "unknown"
+    network_region: str = "unknown"
+    estimated_cost_per_1k_tokens: float | None = None
+
+
+@dataclass(frozen=True)
+class BenchmarkConfig:
+    """Execution controls for warm-up, concurrency, and optional rate limiting."""
+
+    warmup_runs: int = 0
+    measured_runs: int = 1
+    concurrency: int = 1
+    requests_per_second: float | None = None
+
+    def __post_init__(self) -> None:
+        if min(self.warmup_runs, self.measured_runs) < 0:
+            raise ValueError("run counts cannot be negative")
+        if self.concurrency < 1:
+            raise ValueError("concurrency must be positive")
+        if self.requests_per_second is not None and self.requests_per_second <= 0:
+            raise ValueError("requests_per_second must be positive")
+
+
+class ScenarioBenchmarkRunner:
+    """Run identical scenarios concurrently and report contract outcomes."""
+
+    def __init__(self, clock_runner: BenchmarkRunner | None = None):
+        self.runner = clock_runner or BenchmarkRunner()
+
+    def run(
+        self,
+        provider: StreamingProvider,
+        scenario: BenchmarkScenario,
+        plan: InferencePlan,
+        config: BenchmarkConfig | None = None,
+        quality_evaluator: Callable[[str], float] | None = None,
+    ) -> dict[str, object]:
+        config = config or BenchmarkConfig()
+        for _ in range(config.warmup_runs):
+            self.runner.run_streaming(provider, scenario.prompt, plan)
+        interval = 1 / config.requests_per_second if config.requests_per_second else 0
+
+        def measure(_: int) -> tuple[LatencySample, float | None]:
+            if interval:
+                sleep(interval)
+            result = self.runner.run_streaming(provider, scenario.prompt, plan)
+            quality = quality_evaluator(result.text) if quality_evaluator else None
+            return result.sample, quality
+
+        with ThreadPoolExecutor(max_workers=config.concurrency) as executor:
+            measured = list(executor.map(measure, range(config.measured_runs)))
+        samples = [item[0] for item in measured]
+        summary = self._summary(samples, scenario, measured)
+        return summary
+
+    @staticmethod
+    def _summary(
+        samples: list[LatencySample],
+        scenario: BenchmarkScenario,
+        measured: list[tuple[LatencySample, float | None]],
+    ) -> dict[str, object]:
+        successful = [sample for sample in samples if sample.success]
+        quality_scores = [score for _, score in measured if score is not None]
+        deadline_violations = sum(
+            sample.success and sample.end_to_end_ms > scenario.contract.deadline_ms
+            for sample in samples
+        )
+        tokens = scenario.profile.prompt_tokens + scenario.profile.expected_output_tokens
+        cost = (
+            tokens / 1000 * scenario.estimated_cost_per_1k_tokens
+            if scenario.estimated_cost_per_1k_tokens is not None else None
+        )
+        from .metrics import percentile
+        return {
+            "scenario": scenario.name,
+            "runs": len(samples),
+            "warmup_excluded": True,
+            "concurrency": len(samples),
+            "success_rate": len(successful) / len(samples) if samples else 0.0,
+            "ttft_p50_ms": percentile([s.ttft_ms for s in samples], 50) if samples else None,
+            "ttft_p95_ms": percentile([s.ttft_ms for s in samples], 95) if samples else None,
+            "tpot_p50_ms": percentile([s.tpot_ms for s in successful if s.tpot_ms is not None], 50) if any(s.tpot_ms is not None for s in successful) else None,
+            "e2e_p50_ms": percentile([s.end_to_end_ms for s in samples], 50) if samples else None,
+            "e2e_p95_ms": percentile([s.end_to_end_ms for s in samples], 95) if samples else None,
+            "deadline_violations": deadline_violations,
+            "quality_mean": sum(quality_scores) / len(quality_scores) if quality_scores else None,
+            "quality_floor": scenario.contract.quality_floor,
+            "quality_satisfied": all(score >= scenario.contract.quality_floor for score in quality_scores) if quality_scores else None,
+            "estimated_cost": cost,
+            "cache_state": scenario.cache_state,
+            "network_region": scenario.network_region,
+        }

latencyops/capabilities.py

@@ -0,0 +1,66 @@
+"""Provider capability negotiation and safe plan enforcement."""
+
+from dataclasses import dataclass
+
+from .models import InferencePlan, LatencyContract
+
+
+@dataclass(frozen=True)
+class ProviderCapabilities:
+    """Execution features a provider can honor."""
+
+    streaming: bool = True
+    model_mapping: bool = True
+    precision: bool = False
+    context_policy: bool = False
+    speculation: bool = False
+    early_exit: bool = False
+    usage_metadata: bool = False
+    queue_metrics: bool = False
+    cache_metrics: bool = False
+
+
+@dataclass(frozen=True)
+class EnforcedPlan:
+    """Requested plan, provider-supported plan, and unapplied choices."""
+
+    requested: InferencePlan
+    enforced: InferencePlan
+    unsupported: tuple[str, ...] = ()
+
+
+def enforce_plan(
+    contract: LatencyContract,
+    plan: InferencePlan,
+    capabilities: ProviderCapabilities,
+) -> EnforcedPlan:
+    """Remove unsupported optimizations while preserving critical safeguards."""
+    unsupported: list[str] = []
+    precision = plan.precision
+    context_policy = plan.context_policy
+    speculative_tokens = plan.speculative_tokens
+    early_exit = plan.early_exit
+    rationale = list(plan.rationale)
+    if precision != "fp16" and not capabilities.precision:
+        unsupported.append("precision")
+        precision = "fp16"
+    if context_policy != "full" and not capabilities.context_policy:
+        unsupported.append("context_policy")
+        context_policy = "full"
+    if speculative_tokens and not capabilities.speculation:
+        unsupported.append("speculation")
+        speculative_tokens = 0
+    if early_exit and not capabilities.early_exit:
+        unsupported.append("early_exit")
+        early_exit = False
+    if unsupported:
+        rationale.append("provider cannot enforce: " + ", ".join(unsupported))
+    enforced = InferencePlan(
+        plan.model_tier, precision, context_policy, speculative_tokens, early_exit, tuple(rationale)
+    )
+    if contract.risk_class == "critical" and enforced != plan:
+        enforced = InferencePlan(
+            plan.model_tier, "fp16", "full", 0, False,
+            tuple(rationale + ["critical safeguards enforced at provider boundary"]),
+        )
+    return EnforcedPlan(plan, enforced, tuple(unsupported))

latencyops/cli.py

@@ -0,0 +1,54 @@
+"""Command-line entry point for the LatencyOps reference application."""
+
+import argparse
+import json
+from dataclasses import asdict
+
+from .config import build_service, load_config
+from .models import LatencyContract, RequestProfile, SystemSignals
+from .policy import ElasticInferencePlanner
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="latencyops")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+    gateway = subparsers.add_parser("gateway", help="start the configured HTTP gateway")
+    gateway.add_argument("--config", required=True)
+    plan = subparsers.add_parser("plan", help="calculate a plan from a JSON request")
+    plan.add_argument("--request", required=True)
+    health = subparsers.add_parser("health", help="check configured provider health")
+    health.add_argument("--config", required=True)
+    args = parser.parse_args(argv)
+    if args.command == "plan":
+        with open(args.request, encoding="utf-8") as handle:
+            request = json.load(handle)
+        contract = LatencyContract(
+            deadline_ms=float(request.get("deadline_ms", 1000)),
+            quality_floor=float(request.get("quality_floor", 0.9)),
+            risk_class=request.get("risk_class", "medium"),
+        )
+        profile = RequestProfile(
+            prompt_tokens=int(request.get("prompt_tokens", 0)),
+            expected_output_tokens=int(request.get("expected_output_tokens", 16)),
+            difficulty=float(request.get("difficulty", 0.5)),
+            cache_pressure=float(request.get("cache_pressure", 0)),
+            queue_pressure=float(request.get("queue_pressure", 0)),
+        )
+        print(json.dumps(asdict(ElasticInferencePlanner().plan(contract, profile, SystemSignals(
+            queue_pressure=profile.queue_pressure, cache_pressure=profile.cache_pressure,
+        ))), indent=2))
+        return 0
+    config = load_config(args.config)
+    if args.command == "health":
+        results = {name: provider.health().__dict__ if hasattr(provider, "health") else {"healthy": True, "detail": "not configured"} for name, provider in config.providers.items()}
+        print(json.dumps(results, indent=2))
+        return 0 if all(result["healthy"] for result in results.values()) else 1
+    service = build_service(config)
+    from .gateway import create_server
+    print(f"LatencyOps gateway listening on http://{config.host}:{config.port}")
+    create_server(config.host, config.port, service).serve_forever()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

latencyops/comparison.py

@@ -0,0 +1,52 @@
+"""Benchmark comparison and constraint-satisfaction helpers."""
+
+from dataclasses import dataclass
+from typing import Iterable
+
+from .benchmark import BenchmarkRunner
+from .metrics import summarize
+from .models import InferencePlan, LatencyContract, LatencySample, RequestProfile
+from .router import StreamingProvider
+
+
+@dataclass(frozen=True)
+class WorkloadCase:
+    """Synthetic workload metadata; prompt content is never included in reports."""
+
+    name: str
+    prompt: str
+    contract: LatencyContract
+    profile: RequestProfile
+
+
+def run_workload_matrix(
+    provider: StreamingProvider,
+    cases: Iterable[WorkloadCase],
+    plan: InferencePlan,
+    runner: BenchmarkRunner | None = None,
+) -> list[dict[str, object]]:
+    """Run cases and report measurements plus contract satisfaction."""
+    runner = runner or BenchmarkRunner()
+    rows = []
+    for case in cases:
+        result = runner.run_streaming(provider, case.prompt, plan)
+        sample = result.sample
+        rows.append({
+            "case": case.name,
+            "success": sample.success,
+            "ttft_ms": sample.ttft_ms,
+            "tpot_ms": sample.tpot_ms,
+            "end_to_end_ms": sample.end_to_end_ms,
+            "deadline_satisfied": sample.success and sample.end_to_end_ms <= case.contract.deadline_ms,
+            "quality_floor": case.contract.quality_floor,
+        })
+    return rows
+
+
+def compare(samples_by_provider: dict[str, list[LatencySample]]) -> list[dict[str, object]]:
+    """Return stable, ranked summaries without retaining prompts or responses."""
+    rows = []
+    for provider, samples in samples_by_provider.items():
+        summary = summarize(samples)
+        rows.append({"provider": provider, **summary})
+    return sorted(rows, key=lambda row: (row.get("e2e_p95_ms", float("inf")), row["provider"]))

latencyops/config.py

@@ -0,0 +1,71 @@
+"""TOML configuration for the installable LatencyOps application."""
+
+import os
+import tomllib
+from dataclasses import dataclass
+from pathlib import Path
+
+from .adapters import OpenAIChatCompatibleProvider
+from .gateway import GatewayService
+from .models import ProviderCandidate
+from .policy import ElasticInferencePlanner
+from .router import ModelRouter
+from .telemetry import PrometheusExporter, TelemetryRecorder
+
+
+@dataclass(frozen=True)
+class AppConfig:
+    """Validated runtime configuration loaded from TOML and environment variables."""
+
+    host: str
+    port: int
+    providers: dict[str, OpenAIChatCompatibleProvider]
+    candidates: list[ProviderCandidate]
+
+
+def load_config(path: str | Path) -> AppConfig:
+    with Path(path).open("rb") as handle:
+        raw = tomllib.load(handle)
+    server = raw.get("server", {})
+    provider_table = raw.get("providers", {})
+    providers: dict[str, OpenAIChatCompatibleProvider] = {}
+    candidates: list[ProviderCandidate] = []
+    for name, settings in provider_table.items():
+        endpoint = settings.get("endpoint")
+        key_env = settings.get("api_key_env")
+        if not endpoint or not key_env:
+            raise ValueError(f"provider {name!r} requires endpoint and api_key_env")
+        key = os.environ.get(key_env)
+        if not key:
+            raise ValueError(f"environment variable {key_env!r} is not set")
+        models = settings.get("models", {})
+        provider = OpenAIChatCompatibleProvider(
+            name,
+            endpoint,
+            api_key=key,
+            timeout=float(settings.get("timeout_seconds", 30)),
+            model_names=models,
+            chat_template_kwargs=settings.get("chat_template_kwargs"),
+            health_endpoint=settings.get("health_endpoint"),
+        )
+        providers[name] = provider
+        for tier, model in models.items():
+            candidates.append(ProviderCandidate(name, tier, True, cost_per_1k_tokens=settings.get("cost_per_1k_tokens")))
+    missing = {"small", "standard", "large"} - {candidate.model_tier for candidate in candidates}
+    if missing:
+        raise ValueError(f"configuration is missing model tiers: {sorted(missing)}")
+    return AppConfig(str(server.get("host", "127.0.0.1")), int(server.get("port", 8080)), providers, candidates)
+
+
+def build_service(config: AppConfig) -> GatewayService:
+    exporter = PrometheusExporter()
+    telemetry = TelemetryRecorder([exporter])
+    lookup = {candidate.name: config.providers[candidate.name] for candidate in config.candidates}
+    by_tier = {
+        candidate.model_tier: config.providers[candidate.name]
+        for candidate in config.candidates
+    }
+    return GatewayService(
+        ModelRouter(by_tier), ElasticInferencePlanner(), telemetry,
+        provider_candidates=config.candidates, provider_lookup=lookup,
+    )