tokenbiryani 0.2.0__py3-none-any.whl

tokenbiryani/core/gateway.py

@@ -0,0 +1,2395 @@
+"""The gateway: admit, route, lease, proxy, reconcile, recover.
+
+This is the seven-stage pipeline from the plan. The two subtleties worth reading for
+are the lease lifecycle (reserve before dispatch, reconcile against real usage) and
+the streaming commit boundary in ``stream`` — the single point where a failure stops
+being hideable.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import os
+import random
+import re
+import time
+import uuid
+from dataclasses import dataclass, replace
+from typing import Any, AsyncIterator, Dict, List, Mapping, Optional, Tuple
+
+import httpx
+
+from ..config import AccountConfig, Config, ConfigError, KeyConfig
+from ..observability.events import Attempt, EventLog, RequestEvent
+from ..observability.usage import (
+    aggregate,
+    resolve_bucket,
+    resolve_window,
+    sample_from_event,
+)
+from ..providers.anthropic_api import build_upstream
+from ..providers.base import BLOCKED_RESPONSE_HEADERS, Upstream
+from ..providers.translate import map_model
+from ..proxy import sse
+from ..proxy.errors import (
+    AccountAction,
+    Classification,
+    RequestAction,
+    classify,
+    classify_exception,
+)
+from ..store.base import (
+    SCOPE_ACCOUNT,
+    SCOPE_KEY,
+    SCOPE_SESSION,
+    SCOPE_SESSION_TURNS,
+    StateStore,
+    build_store,
+)
+from . import batch as batch_lane
+from .account import AccountRuntime, AccountState, Usage
+from .breaker import CircuitBreaker
+from .cacheadvice import (
+    CacheAdvisor,
+    cacheable_prefix_tokens,
+    has_cache_breakpoint,
+    insert_cache_breakpoint,
+)
+from .diagnostics import inspect_headers, looks_unified
+from .estimator import OutputEstimator
+from .keys import KeyRegistry, generate_key, record_from_config
+from .limits import LimitMirror, TokenEstimate, estimate_request
+from .oauth import OAuthClient, OAuthError, OAuthTokens, PendingLogin, make_verifier
+from .pacing import PaceReading, PacingGovernor
+from .pacing import week_start as _pacing_week_start
+from .queue import (
+    MAX_WAIT_HEADER,
+    PRIORITY_BATCH,
+    PRIORITY_HEADER,
+    CapacityGate,
+    QueueFull,
+    admission_check,
+    parse_priority,
+    priority_name,
+)
+from .router import Decision, Router, RoutingContext, available_strategies
+from .secrets import SecretBox, SecretError
+from .session import session_key as compute_session_key
+
+MESSAGES_PATH = "/v1/messages"
+
+logger = logging.getLogger("tokenbiryani")
+
+#: Account ids appear in URLs, metric labels and log lines; keep them boring.
+_SAFE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$")
+
+#: Never leaves the process. Stripped from every account payload the API returns.
+_CREDENTIAL_FIELDS = ("api_key", "oauth_access", "oauth_refresh")
+
+
+class GatewayError(Exception):
+    """A failure the gateway itself is reporting, in Anthropic's error shape."""
+
+    def __init__(
+        self,
+        status: int,
+        message: str,
+        kind: str = "api_error",
+        retry_after: Optional[float] = None,
+        extra_headers: Optional[Dict[str, str]] = None,
+    ) -> None:
+        super().__init__(message)
+        self.status = status
+        self.message = message
+        self.kind = kind
+        self.retry_after = retry_after
+        self.extra_headers = extra_headers or {}
+
+    def payload(self) -> Dict[str, Any]:
+        return {"type": "error", "error": {"type": self.kind, "message": self.message}}
+
+    def headers(self) -> Dict[str, str]:
+        headers = dict(self.extra_headers)
+        if self.retry_after is not None:
+            headers["retry-after"] = str(int(max(1, round(self.retry_after))))
+        return headers
+
+
+class _SpillToBatch(Exception):
+    """Internal: the pool is saturated and this request is allowed to spill."""
+
+
+@dataclass
+class Completion:
+    status: int
+    headers: Dict[str, str]
+    content: bytes
+    event: RequestEvent
+
+
+def _response_headers(upstream_headers: Mapping[str, str], account_id: str) -> Dict[str, str]:
+    out = {
+        k: v for k, v in upstream_headers.items() if k.lower() not in BLOCKED_RESPONSE_HEADERS
+    }
+    out["x-tokenbiryani-account"] = account_id
+    return out
+
+
+#: How long a computed pace is reused for. A quota window is days long; a request
+#: does not need a fresher number than this, and pacing reads the spend ledger.
+PACE_CACHE_SECONDS = 5.0
+
+
+def _pacing_settings(config: Config) -> Dict[str, Any]:
+    pacing = config.pacing
+    return {
+        "enabled": pacing.enabled,
+        "mode": pacing.mode,
+        "window": pacing.window,
+        "curve": pacing.curve,
+        "ahead_threshold": pacing.ahead_threshold,
+        "behind_threshold": pacing.behind_threshold,
+        "max_batch_delay_seconds": pacing.max_batch_delay_seconds,
+        "weekly_budget_usd": pacing.weekly_budget_usd,
+    }
+
+
+def _build_pacing(config: Config) -> PacingGovernor:
+    return PacingGovernor(**_pacing_settings(config))
+
+
+def _build_estimator(config: Config) -> OutputEstimator:
+    """One estimator per gateway, configured from `routing.*`."""
+    routing = config.routing
+    return OutputEstimator(
+        mode=routing.output_estimate,
+        quantile=routing.output_estimate_quantile,
+        min_samples=routing.output_estimate_min_samples,
+        window=routing.output_estimate_window,
+        floor=routing.output_estimate_floor,
+        max_undershoot=routing.output_estimate_max_undershoot,
+    )
+
+
+class Gateway:
+    def __init__(
+        self,
+        config: Config,
+        store: Optional[StateStore] = None,
+        client: Optional[httpx.AsyncClient] = None,
+    ) -> None:
+        self.config = config
+        self.store = store or build_store(config)
+        self.keys = KeyRegistry(config.keys)
+        self.events = EventLog(config.observability.event_buffer)
+        self.gate = CapacityGate(config.queue.max_size, config.queue.default_max_wait_seconds)
+        self.router = Router(config.routing.strategy, config.routing.weights)
+        self.estimator = _build_estimator(config)
+        self.cache_advisor = CacheAdvisor(
+            min_requests=config.cache.advice_min_requests
+        )
+        self.pacing = _build_pacing(config)
+        #: Pacing reads the spend ledger, which is a store round trip. A request
+        #: does not need a fresher answer than this — a quota window is days long.
+        self._pace_value: Optional[float] = None
+        self._pace_cached_at: float = 0.0
+
+        self.accounts: Dict[str, AccountRuntime] = {}
+        self.upstreams: Dict[str, Upstream] = {}
+        for account_config in config.accounts:
+            self.accounts[account_config.id] = AccountRuntime(
+                config=account_config,
+                mirror=_mirror_for(account_config),
+                breaker=CircuitBreaker(
+                    failure_threshold=config.breaker.failure_threshold,
+                    cooldown_seconds=config.breaker.cooldown_seconds,
+                ),
+            )
+            self.upstreams[account_config.id] = self._make_upstream(account_config)
+
+        #: Accounts declared in the config file. They belong to the file: the API
+        #: can neither edit nor delete them, only accounts it created itself.
+        self._config_account_ids = {a.id for a in config.accounts}
+        self._secrets: Optional[SecretBox] = None
+        self._oauth: Optional[OAuthClient] = None
+        #: Decrypted access tokens, held in memory only. The request path reads
+        #: these synchronously; the refresh task is what keeps them current.
+        self._oauth_tokens: Dict[str, str] = {}
+        #: Session facts the console shows — expiry, whether it can be renewed.
+        #: Kept beside the token so a snapshot needs no store round-trip.
+        self._oauth_meta: Dict[str, Dict[str, Any]] = {}
+        self._logins: Dict[str, PendingLogin] = {}
+        #: Console edits to accounts the config file declares. The file keeps the
+        #: credential and the identity; these are the operational fields an operator
+        #: needs to change at 3am without opening an editor on the server.
+        self._account_overrides: Dict[str, Dict[str, Any]] = {}
+        #: Console edits to gateway-wide settings, applied over the file's values.
+        self._settings: Dict[str, Any] = {}
+        #: What the file asked for, kept because a console override replaces the
+        #: live value and the Settings screen shows both.
+        self._file_pricing_table = config.pricing_table
+
+        self._client = client
+        self._owns_client = client is None
+
+    # ---- lifecycle -----------------------------------------------------------
+
+    async def startup(self) -> None:
+        if self._client is None:
+            self._client = httpx.AsyncClient(
+                timeout=httpx.Timeout(self.config.server.request_timeout_seconds, connect=10.0)
+            )
+        await self.store.startup()
+        await self.refresh_settings()
+        await self.refresh_accounts()
+        await self.hydrate_spend()
+        await self.refresh_keys()
+
+    async def hydrate_spend(self) -> None:
+        """Load each account's windowed spend from the store.
+
+        Account caps are checked on the synchronous routing path, so the figure is
+        cached in memory. Without this, a restart would hand every capped account a
+        clean slate — which is the whole reason the ledger is persistent.
+        """
+        totals = await self.store.spend_by_scope(SCOPE_ACCOUNT, self.config.spend.window_seconds)
+        for account_id, total in totals.items():
+            account = self.accounts.get(account_id)
+            if account is not None:
+                account.spend_usd = total
+
+    async def resync_spend(self, interval: float = 60.0) -> None:
+        """Re-read spend and managed keys so this instance cannot drift from the store."""
+        while True:
+            await asyncio.sleep(interval)
+            try:
+                await self.hydrate_spend()
+                await self.refresh_keys()
+                await self.refresh_accounts()
+            except Exception as exc:  # noqa: BLE001 - a store blip must not kill routing
+                logger.warning("state resync failed: %s", exc)
+
+    async def refresh_keys(self) -> None:
+        """Load managed keys from the store into the registry."""
+        self.keys.set_managed(await self.store.list_keys())
+
+    # ---- managed accounts ----------------------------------------------------
+
+    @property
+    def secrets(self) -> SecretBox:
+        if self._secrets is None:
+            try:
+                self._secrets = SecretBox(key_path=self.config.store.secret_key_path)
+            except (SecretError, OSError) as exc:
+                # Reached the first time an account is stored, which is a request the
+                # operator is making from the console. A missing extra or an
+                # unwritable key path is their problem to fix, so say which it is
+                # rather than returning an opaque 500.
+                raise GatewayError(
+                    503,
+                    f"cannot store credentials: {exc}",
+                    kind="api_error",
+                ) from exc
+        return self._secrets
+
+    def is_config_account(self, account_id: str) -> bool:
+        return account_id in self._config_account_ids
+
+    # ---- overrides on config accounts ----------------------------------------
+
+    #: What the console may change on an account the file declares. Credentials,
+    #: type and base_url are absent on purpose: those are the file's business, and
+    #: an override that could repoint an account at another host would make the
+    #: config file a lie rather than an authority.
+    OVERRIDABLE = (
+        "name", "enabled", "cost_tier", "priority", "spend_cap_usd",
+        "models", "max_concurrency",
+    )
+
+    @staticmethod
+    def is_override(record: Mapping[str, Any]) -> bool:
+        return str(record.get("kind") or "") == "override"
+
+    def file_config(self, account_id: str) -> Optional[AccountConfig]:
+        """The account exactly as the config file declares it."""
+        return next((a for a in self.config.accounts if a.id == account_id), None)
+
+    def _apply_override(self, account_id: str) -> None:
+        """Point the runtime at the file's account, or at a copy carrying the override.
+
+        A copy, never an edit in place: `AccountRuntime.config` starts out as the
+        very object the file was parsed into, and mutating it would destroy the
+        value an override has to be revertible to. With a copy, reverting is just
+        pointing back at the original.
+        """
+        runtime = self.accounts.get(account_id)
+        base = self.file_config(account_id)
+        if runtime is None or base is None:
+            return
+        fields = {
+            name: value
+            for name, value in (self._account_overrides.get(account_id) or {}).items()
+            if name in self.OVERRIDABLE and value is not None
+        }
+        runtime.config = replace(base, **fields) if fields else base
+        runtime.mirror.observable = runtime.config.observable_limits
+
+    def _apply_all_overrides(self) -> None:
+        for account_id in self._config_account_ids:
+            self._apply_override(account_id)
+
+    def override_record(self, account_id: str, fields: Mapping[str, Any]) -> Dict[str, Any]:
+        return {
+            "id": account_id,
+            "kind": "override",
+            "fields": {k: v for k, v in fields.items() if k in self.OVERRIDABLE},
+            "updated_at": time.time(),
+        }
+
+    # ---- gateway-wide settings -----------------------------------------------
+
+    #: Settings the console may change without an editor, and what each is checked
+    #: against. Everything else stays a config-file decision.
+    SETTABLE = ("strategy", "pricing_source")
+
+    async def refresh_settings(self) -> None:
+        """Load console-set settings from the store and apply them over the file."""
+        try:
+            stored = await self.store.get_settings()
+        except Exception as exc:  # noqa: BLE001 - a store blip must not stop startup
+            logger.warning("could not read settings: %s", exc)
+            return
+        self._settings = {k: v for k, v in (stored or {}).items() if k in self.SETTABLE}
+        self.apply_settings()
+
+    def apply_settings(self) -> None:
+        """Re-apply stored settings. Called after anything that rebuilds from config.
+
+        A reload reads the file, so without this a console-set strategy would
+        silently revert the next time anyone touched tokenbiryani.yaml.
+        """
+        strategy = self._settings.get("strategy")
+        if strategy and strategy != self.router.strategy:
+            # A whole Router, not just its name: the scorer and the weight spec are
+            # chosen by the strategy, so renaming the field in place would leave the
+            # gateway claiming round_robin while still scoring like sticky_headroom.
+            self.router = Router(str(strategy), self.config.routing.weights)
+        source = self._settings.get("pricing_source")
+        if source:
+            self.config.use_pricing(str(source))
+
+    async def update_settings(self, changes: Mapping[str, Any]) -> Dict[str, Any]:
+        """Change a setting from the console. Persisted, so a restart keeps it."""
+        unknown = [k for k in changes if k not in self.SETTABLE]
+        if unknown:
+            raise GatewayError(
+                400,
+                "unknown setting(s): {}. Settable: {}".format(
+                    ", ".join(sorted(unknown)), ", ".join(self.SETTABLE)
+                ),
+                kind="invalid_request_error",
+            )
+        if "strategy" in changes:
+            strategy = str(changes["strategy"] or "")
+            if strategy not in available_strategies():
+                raise GatewayError(
+                    400,
+                    "unknown strategy {!r}. Available: {}".format(
+                        strategy, ", ".join(sorted(available_strategies()))
+                    ),
+                    kind="invalid_request_error",
+                )
+        if "pricing_source" in changes:
+            source = str(changes["pricing_source"] or "")
+            if source not in ("builtin", "config"):
+                raise GatewayError(
+                    400,
+                    "pricing_source must be 'builtin' (the dated table shipped with "
+                    "this release) or 'config' (only what tokenbiryani.yaml names)",
+                    kind="invalid_request_error",
+                )
+        for name, value in changes.items():
+            self._settings[name] = value
+            await self.store.put_setting(name, value)
+        self.apply_settings()
+        return self.settings_view()
+
+    def settings_view(self) -> Dict[str, Any]:
+        """What the Settings screen edits, and where each value came from."""
+        return {
+            "strategy": {
+                "value": self.router.strategy,
+                "source": "console" if "strategy" in self._settings else "config",
+                "file_value": self.config.routing.strategy,
+                "options": sorted(available_strategies()),
+            },
+            "pricing_source": {
+                "value": self.config.pricing_table,
+                "source": "console" if "pricing_source" in self._settings else "config",
+                "file_value": self._file_pricing_table,
+                "options": ["builtin", "config"],
+            },
+            "priced_models": sorted(self.config.pricing),
+            "priced_as_of": self.config.pricing_as_of,
+            "priced_source": self.config.pricing_source,
+        }
+
+    def _make_upstream(self, account_config: AccountConfig) -> Upstream:
+        """One construction point, so an oauth account always gets its token source."""
+        if account_config.type == "oauth":
+            account_id = account_config.id
+            return build_upstream(account_config, lambda: self.oauth_token(account_id))
+        return build_upstream(account_config)
+
+    def _config_from_record(self, record: Dict[str, Any]) -> AccountConfig:
+        return AccountConfig(
+            id=str(record["id"]),
+            name=str(record.get("name") or ""),
+            type=str(record.get("type") or "anthropic_api"),
+            api_key=self.secrets.decrypt(str(record.get("api_key") or "")),
+            base_url=str(record.get("base_url") or ""),
+            priority=float(record.get("priority") or 0.0),
+            cost_tier=float(record.get("cost_tier") or 1.0),
+            models=list(record.get("models") or ["*"]),
+            max_concurrency=int(record.get("max_concurrency") or 16),
+            spend_cap_usd=record.get("spend_cap_usd"),
+            enabled=bool(record.get("enabled", True)),
+            headers=dict(record.get("headers") or {}),
+            options=dict(record.get("options") or {}),
+            observable_limits=bool(record.get("observable_limits", True)),
+            assumed_headroom=float(record.get("assumed_headroom") or 0.5),
+        )
+
+    def _cache_session(self, record: Dict[str, Any]) -> None:
+        """Decrypt a stored subscription session into the in-memory token cache."""
+        if record.get("type") != "oauth":
+            return
+        account_id = str(record.get("id") or "")
+        self._oauth_meta[account_id] = {
+            "session_expires_at": record.get("oauth_expires_at"),
+            "can_refresh": bool(record.get("oauth_refresh")),
+        }
+        try:
+            self._oauth_tokens[account_id] = self.secrets.decrypt(
+                str(record.get("oauth_access") or "")
+            )
+        except SecretError as exc:
+            logger.warning("cannot read the session for %r: %s", account_id, exc)
+            self._oauth_tokens.pop(account_id, None)
+
+    async def refresh_accounts(self) -> None:
+        """Merge accounts stored through the API into the running pool.
+
+        Config accounts always win a name clash: the file is the operator's, and a
+        record in a shared store must not be able to redirect it.
+        """
+        records = await self.store.list_accounts()
+        seen = set()
+        overrides: Dict[str, Dict[str, Any]] = {}
+        for record in records:
+            account_id = str(record.get("id") or "")
+            if not account_id:
+                continue
+            if self.is_override(record):
+                # An override belongs to a config account and is not one itself.
+                fields = record.get("fields")
+                overrides[account_id] = dict(fields) if isinstance(fields, dict) else {}
+                continue
+            if account_id in self._config_account_ids:
+                continue
+            seen.add(account_id)
+            try:
+                account_config = self._config_from_record(record)
+            except SecretError as exc:
+                logger.warning("account %r is unreadable: %s", account_id, exc)
+                continue
+            # Decrypt the session now, not on the request path.
+            self._cache_session(record)
+            existing = self.accounts.get(account_id)
+            if existing is None:
+                self.accounts[account_id] = AccountRuntime(
+                    config=account_config,
+                    mirror=_mirror_for(account_config),
+                    breaker=CircuitBreaker(
+                        failure_threshold=self.config.breaker.failure_threshold,
+                        cooldown_seconds=self.config.breaker.cooldown_seconds,
+                    ),
+                )
+                self.upstreams[account_id] = self._make_upstream(account_config)
+                continue
+            rotated = (
+                existing.config.api_key != account_config.api_key
+                or existing.config.base_url != account_config.base_url
+                or existing.config.type != account_config.type
+            )
+            existing.config = account_config
+            existing.mirror.observable = account_config.observable_limits
+            if rotated:
+                self.upstreams[account_id] = self._make_upstream(account_config)
+                existing.disabled_reason = None
+                existing.breaker.record_success()
+
+        # Anything managed that has gone from the store has been deleted elsewhere.
+        for account_id in list(self.accounts):
+            if account_id not in self._config_account_ids and account_id not in seen:
+                self.accounts.pop(account_id, None)
+                self.upstreams.pop(account_id, None)
+
+        # An override that has been dropped must put the file's own value back, so
+        # reverting is a delete rather than a second edit guessing what the file said.
+        self._account_overrides = overrides
+        self._apply_all_overrides()
+
+    def account_record(
+        self, account_id: str, name: str, api_key: str, **options: Any
+    ) -> Dict[str, Any]:
+        return {
+            "id": account_id,
+            "name": name or account_id,
+            "type": str(options.get("type") or "anthropic_api"),
+            "api_key": self.secrets.encrypt(api_key),
+            "base_url": str(options.get("base_url") or ""),
+            "cost_tier": float(options.get("cost_tier") or 1.0),
+            "priority": float(options.get("priority") or 0.0),
+            "models": list(options.get("models") or ["*"]),
+            "max_concurrency": int(options.get("max_concurrency") or 16),
+            "spend_cap_usd": options.get("spend_cap_usd"),
+            "enabled": bool(options.get("enabled", True)),
+            # Forced, not defaulted, for a subscription: an account whose classic
+            # windows never populate would read as permanently full and win every
+            # comparison. Its real budget arrives as unified utilisation instead.
+            "observable_limits": (
+                False
+                if str(options.get("type") or "anthropic_api") == "oauth"
+                else bool(options.get("observable_limits", True))
+            ),
+            "options": dict(options.get("options") or {}),
+            "created_at": time.time(),
+        }
+
+    async def create_account(
+        self, account_id: str, name: str, api_key: str, **options: Any
+    ) -> Dict[str, Any]:
+        account_id = (account_id or "").strip()
+        if not account_id:
+            raise GatewayError(400, "an account needs an id", kind="invalid_request_error")
+        if not _SAFE_ID.match(account_id):
+            raise GatewayError(
+                400,
+                "an account id may contain letters, digits, dashes and underscores only",
+                kind="invalid_request_error",
+            )
+        await self.refresh_accounts()
+        if account_id in self.accounts:
+            raise GatewayError(
+                409, f"an account named {account_id!r} already exists",
+                kind="invalid_request_error",
+            )
+        if not api_key and str(options.get("type") or "anthropic_api") == "anthropic_api":
+            raise GatewayError(400, "an API key is required", kind="invalid_request_error")
+
+        record = self.account_record(account_id, name, api_key, **options)
+        await self.store.put_account(record)
+        await self.refresh_accounts()
+        return self.redacted_account(record)
+
+    async def update_account(self, account_id: str, **changes: Any) -> Dict[str, Any]:
+        if self.is_config_account(account_id):
+            return await self.override_account(account_id, **changes)
+        records = {str(r.get("id")): r for r in await self.store.list_accounts()}
+        record = records.get(account_id)
+        if record is None:
+            raise GatewayError(404, f"no managed account {account_id!r}", kind="not_found_error")
+
+        for field_name in (
+            "name", "base_url", "cost_tier", "priority", "models",
+            "max_concurrency", "spend_cap_usd", "enabled", "observable_limits",
+        ):
+            if field_name in changes and changes[field_name] is not None:
+                record[field_name] = changes[field_name]
+        if changes.get("api_key"):
+            record["api_key"] = self.secrets.encrypt(str(changes["api_key"]))
+
+        await self.store.put_account(record)
+        await self.refresh_accounts()
+
+        # Re-enabling is an operator saying "I fixed it". A disabled_reason from the
+        # last 401, or a breaker still open from the failures that followed, would
+        # otherwise keep the account out of the pool with the toggle reading "on".
+        if changes.get("enabled") is True:
+            runtime = self.accounts.get(account_id)
+            if runtime is not None:
+                runtime.disabled_reason = None
+                runtime.breaker.record_success()
+
+        return self.redacted_account(record)
+
+    async def override_account(self, account_id: str, **changes: Any) -> Dict[str, Any]:
+        """Change an operational field on an account the config file declares.
+
+        The file keeps the credential, the type and the base URL — an override that
+        could repoint an account at another host would make the file a lie. What is
+        left is exactly what an operator needs at 3am: turn it off, rename it,
+        re-tier it, cap it. The change is stored in the gateway, not written back to
+        the file, and the console says so; deleting the override puts the file's own
+        values back.
+        """
+        blank: Tuple[Any, ...] = (None, "", {}, ())
+        rejected = [
+            name for name in changes
+            if name not in self.OVERRIDABLE and changes[name] not in blank
+        ]
+        if rejected:
+            raise GatewayError(
+                409,
+                "{!r} is declared in tokenbiryani.yaml. The console can change {} on "
+                "it; {} must be edited in the file.".format(
+                    account_id,
+                    ", ".join(self.OVERRIDABLE),
+                    ", ".join(sorted(rejected)),
+                ),
+                kind="invalid_request_error",
+            )
+        existing = dict(self._account_overrides.get(account_id) or {})
+        existing.update({
+            name: value for name, value in changes.items()
+            if name in self.OVERRIDABLE and value is not None
+        })
+        await self.store.put_account(self.override_record(account_id, existing))
+        await self.refresh_accounts()
+
+        runtime = self.accounts.get(account_id)
+        if runtime is not None and changes.get("enabled") is True:
+            runtime.disabled_reason = None
+            runtime.breaker.record_success()
+        return self.account_detail(account_id) or {"id": account_id}
+
+    async def clear_override(self, account_id: str) -> bool:
+        """Drop a config account's override, restoring what the file says."""
+        removed = await self.store.delete_account(account_id)
+        await self.refresh_accounts()
+        return removed
+
+    async def delete_account(self, account_id: str) -> bool:
+        if self.is_config_account(account_id):
+            raise GatewayError(
+                409,
+                f"{account_id!r} is declared in the config file; remove it there and reload",
+                kind="invalid_request_error",
+            )
+        removed = await self.store.delete_account(account_id)
+        # Before the pool is rebuilt: every session pinned here now names a
+        # credential that no longer exists, and leaving them to expire would
+        # report a cache break on each one for the rest of the affinity TTL.
+        # `clear_override` deliberately does not do this — that account survives,
+        # it only stops being overridden.
+        try:
+            dropped = await self.store.clear_affinity_for_account(account_id)
+            if dropped:
+                logger.info("released %d session(s) pinned to %s", dropped, account_id)
+        except Exception:  # noqa: BLE001 - a store blip must not fail the delete
+            logger.warning("could not release affinity for %s", account_id, exc_info=True)
+        await self.refresh_accounts()
+        return removed
+
+    async def test_account(self, account_id: str) -> Dict[str, Any]:
+        """Prove a credential works, before anyone routes real traffic to it."""
+        account = self.accounts.get(account_id)
+        if account is None:
+            raise GatewayError(404, f"no account {account_id!r}", kind="not_found_error")
+        result = await self._probe(self.upstreams[account_id])
+        headers = result.get("headers") or {}
+        if headers:
+            account.mirror.update_from_headers(dict(headers), time.time())
+        classification = result.pop("classification", None)
+        if classification is not None and classification.account_action is AccountAction.DISABLE:
+            account.disabled_reason = classification.kind
+        return result
+
+    async def probe_credential(
+        self, api_key: str, **options: Any
+    ) -> Dict[str, Any]:
+        """Test a credential that has not been stored yet.
+
+        `test_account` is the same probe against an account already in the pool. The
+        reason this second entry point exists is the order of operations: storing
+        first and testing second turns a typo into a `disabled` row somebody has to
+        find and clean up later. Testing first means a wrong key is just a wrong key.
+        """
+        config = AccountConfig(
+            id="probe",
+            type=str(options.get("type") or "anthropic_api"),
+            api_key=api_key,
+            base_url=str(options.get("base_url") or ""),
+            options=dict(options.get("options") or {}),
+        )
+        try:
+            upstream = build_upstream(config)
+        except Exception as exc:  # noqa: BLE001 - a missing adapter extra lands here
+            return {"ok": False, "status": None, "latency": 0.0,
+                    "detail": f"{type(exc).__name__}: {exc}"}
+        return await self._probe(upstream)
+
+    async def _probe(self, upstream: Upstream) -> Dict[str, Any]:
+        """One GET /v1/models, and what it says. Shared by both test entry points."""
+        started = time.time()
+        try:
+            response = await self.client.get(
+                upstream.url("/v1/models"), headers=upstream.request_headers({}), timeout=15.0
+            )
+        except Exception as exc:  # noqa: BLE001 - the reason is the whole answer
+            return {
+                "ok": False, "status": None,
+                "detail": f"{type(exc).__name__}: {exc}",
+                "latency": round(time.time() - started, 3),
+            }
+        latency = round(time.time() - started, 3)
+        result: Dict[str, Any] = {
+            "ok": response.status_code < 300,
+            "status": response.status_code,
+            "latency": latency,
+            "headers": dict(response.headers),
+        }
+        if result["ok"]:
+            result["detail"] = "credential accepted"
+        else:
+            classification = classify(
+                response.status_code, dict(response.headers), _safe_json(response)
+            )
+            result["detail"] = classification.detail or classification.kind
+            result["classification"] = classification
+        return result
+
+    async def diagnose_account(self, account_id: str, spend: bool = False) -> Dict[str, Any]:
+        """Test the credential, and check the rate-limit headers honestly.
+
+        The check itself is the project's largest open risk: if the upstream spells a
+        header differently, the window stays empty, the account reads as full, and
+        routing degrades to round-robin while every meter looks healthy.
+
+        The subtlety is where the headers come from. `GET /v1/models` is free, which
+        makes it the right probe for "does this credential work" — but it carries no
+        `anthropic-ratelimit-*` headers at all, so checking against it reports all
+        nine missing for a perfectly healthy account. Reporting that to someone who
+        has just added their first account would be worse than not checking.
+
+        So there are three answers, not two:
+
+        * ``checked`` — headers from a real ``/v1/messages`` response, either one this
+          account has already served or one this call paid for.
+        * ``not_observed`` — no request has been through this account yet. Nothing is
+          wrong; there is simply nothing to check. Send traffic, or ask for ``spend``.
+        * ``unobservable`` — this upstream reports no limit headers by design.
+        """
+        result = await self.test_account(account_id)
+        result.pop("headers", None)
+        account = self.accounts.get(account_id)
+        result["limits"] = None
+
+        if not result["ok"] or account is None:
+            result["limits_source"] = "unchecked"
+            return result
+
+        observable = bool(account.config.observable_limits)
+        result["observable"] = observable
+        headers: Dict[str, str] = dict(account.last_limit_headers)
+        if not headers and spend:
+            paid = await self._spend_one_request(account_id)
+            if paid is None:
+                result["limits_source"] = "unchecked"
+                result["detail"] = "the check request failed; the credential is fine"
+                return result
+            headers = paid
+        if not headers:
+            # Nothing seen yet. For an unobservable account that is the permanent
+            # answer, not a "come back later".
+            result["limits_source"] = "not_observed" if observable else "unobservable"
+            return result
+        if not observable and not looks_unified(headers):
+            # A subscription session that reports nothing we can read. Holding it to
+            # the API-key checklist would report a fault where there is none.
+            result["limits_source"] = "unobservable"
+            return result
+
+        result["limits"] = inspect_headers(headers, time.time())
+        result["limits_source"] = "checked"
+        return result
+
+    async def _spend_one_request(self, account_id: str) -> Optional[Dict[str, str]]:
+        """One `max_tokens=1` completion, purely to read its headers back.
+
+        The only place this gateway deliberately spends money, and it happens only
+        when a caller has asked for it. `tokenbiryani doctor` is the same trade made
+        from the terminal.
+        """
+        upstream = self.upstreams[account_id]
+        payload = {
+            "model": self._probe_model(),
+            "max_tokens": 1,
+            "messages": [{"role": "user", "content": "hi"}],
+        }
+        try:
+            # Through the adapter, not a raw post: Bedrock and Vertex address the
+            # model in the URL, and a hand-rolled request would miss that.
+            result = await upstream.send(
+                self.client, MESSAGES_PATH, payload,
+                {"content-type": "application/json"}, timeout=30.0,
+            )
+        except Exception as exc:  # noqa: BLE001 - reported, never raised at a console
+            logger.warning("limit-header check failed for %r: %s", account_id, exc)
+            return None
+        if result.status >= 300:
+            return None
+        account = self.accounts.get(account_id)
+        if account is not None:
+            account.observe_headers(dict(result.headers), time.time())
+        return {
+            str(k).lower(): str(v)
+            for k, v in result.headers.items()
+            if str(k).lower().startswith("anthropic-ratelimit-")
+        }
+
+    def _probe_model(self) -> str:
+        """A model this pool is configured for, so the check is not rejected outright."""
+        for name in self.config.pricing:
+            # The bundled table is keyed by prefix (`claude-sonnet-5*`) so that dated
+            # model ids price correctly; the prefix itself is a valid model id.
+            stripped = name.rstrip("*")
+            if stripped and not stripped.endswith("-"):
+                return stripped
+        return "claude-sonnet-5"
+
+    def redacted_account(self, record: Dict[str, Any]) -> Dict[str, Any]:
+        # Every field holding a credential is stripped by name. A subscription's
+        # refresh token is a longer-lived secret than the access token it mints, so
+        # forgetting it here would be the worse of the two leaks.
+        out = {k: v for k, v in record.items() if k not in _CREDENTIAL_FIELDS}
+        out["source"] = "managed"
+        out["has_credential"] = bool(record.get("api_key") or record.get("oauth_access"))
+        if record.get("type") == "oauth":
+            out["session_expires_at"] = record.get("oauth_expires_at")
+            out["can_refresh"] = bool(record.get("oauth_refresh"))
+        return out
+
+    # ---- subscription sessions ----------------------------------------------
+
+    @property
+    def oauth(self) -> OAuthClient:
+        if self._oauth is None:
+            settings = self.config.oauth
+            self._oauth = OAuthClient(
+                client_id=settings.client_id,
+                authorize_url=settings.authorize_url,
+                token_url=settings.token_url,
+                redirect_uri=settings.redirect_uri,
+                scopes=settings.scopes,
+                client=self.client,
+            )
+        return self._oauth
+
+    def oauth_token(self, account_id: str) -> str:
+        """The current access token for a subscription account.
+
+        Synchronous on purpose: it is called from `auth_headers` on the request
+        path. Refreshing happens in the background task, never here — a request
+        must not block on a token endpoint.
+        """
+        token = self._oauth_tokens.get(account_id)
+        if not token:
+            raise GatewayError(
+                401,
+                f"account {account_id!r} has no active session. Log in again from the "
+                "Accounts screen.",
+                kind="authentication_error",
+            )
+        return token
+
+    async def oauth_start(self, account_id: str, name: str = "") -> Dict[str, Any]:
+        """Begin a login. Returns the URL to open and the handle to complete with."""
+        self.oauth.require_configured()
+        account_id = (account_id or "").strip()
+        if not _SAFE_ID.match(account_id):
+            raise GatewayError(
+                400,
+                "an account id may contain letters, digits, dashes and underscores only",
+                kind="invalid_request_error",
+            )
+        await self.refresh_accounts()
+        if account_id in self.accounts:
+            raise GatewayError(
+                409, f"an account named {account_id!r} already exists",
+                kind="invalid_request_error",
+            )
+        login = PendingLogin(
+            state=uuid.uuid4().hex, verifier=make_verifier(),
+            account_id=account_id, name=name or account_id,
+        )
+        now = time.time()
+        # Drop anything abandoned, so a browser closed mid-login cannot accumulate.
+        self._logins = {k: v for k, v in self._logins.items() if not v.expired(now)}
+        self._logins[login.state] = login
+        return {
+            "state": login.state,
+            "authorize_url": self.oauth.authorize_url_for(login),
+            "manual": not self.config.oauth.redirect_uri,
+        }
+
+    async def oauth_complete(self, state: str, code: str) -> Dict[str, Any]:
+        """Finish a login: exchange the code and store the session as an account."""
+        login = self._logins.pop(state, None)
+        if login is None:
+            raise GatewayError(
+                400,
+                "that login is not in progress — it may have expired, or already been "
+                "completed. Start again from the Accounts screen.",
+                kind="invalid_request_error",
+            )
+        if login.expired(time.time()):
+            raise GatewayError(400, "that login expired; start again",
+                               kind="invalid_request_error")
+        code = (code or "").strip()
+        if not code:
+            raise GatewayError(400, "no authorization code was supplied",
+                               kind="invalid_request_error")
+        try:
+            tokens = await self.oauth.exchange(code, login.verifier)
+        except OAuthError as exc:
+            raise GatewayError(400, str(exc), kind="invalid_request_error") from exc
+
+        record = self.account_record(
+            login.account_id, login.name, "",
+            type="oauth",
+            # Nothing to reserve against and nothing to read: a subscription session
+            # sends no rate-limit headers.
+            observable_limits=False,
+        )
+        self._store_session(record, tokens)
+        await self.store.put_account(record)
+        await self.refresh_accounts()
+        return self.redacted_account(record)
+
+    def _store_session(self, record: Dict[str, Any], tokens: OAuthTokens) -> None:
+        record["oauth_access"] = self.secrets.encrypt(tokens.access_token)
+        record["oauth_refresh"] = self.secrets.encrypt(tokens.refresh_token)
+        record["oauth_expires_at"] = tokens.expires_at
+        record["oauth_scope"] = tokens.scope
+
+    async def refresh_oauth_sessions(self) -> None:
+        """Renew subscription tokens before they expire.
+
+        Reading a file the Claude CLI keeps fresh covers a single session and nothing
+        else. A gateway holding several has no CLI to lean on, so it renews them
+        itself.
+        """
+        settings = self.config.oauth
+        now = time.time()
+        for record in await self.store.list_accounts():
+            if record.get("type") != "oauth":
+                continue
+            account_id = str(record.get("id") or "")
+            expires_at = record.get("oauth_expires_at")
+            if not isinstance(expires_at, (int, float)):
+                # Unknown expiry: nothing to renew against. The token is used until
+                # the upstream rejects it, which the error taxonomy already handles.
+                continue
+            if float(expires_at) - settings.refresh_skew_seconds > now:
+                continue
+            try:
+                refresh_token = self.secrets.decrypt(str(record.get("oauth_refresh") or ""))
+                tokens = await self.oauth.refresh(refresh_token)
+            except (OAuthError, SecretError) as exc:
+                # Disable rather than retry forever: an expired session that cannot
+                # be renewed needs a human, and silently failing every request while
+                # reading "ready" is the outcome worth avoiding.
+                logger.warning("could not refresh session for %r: %s", account_id, exc)
+                runtime = self.accounts.get(account_id)
+                if runtime is not None:
+                    runtime.disabled_reason = "session expired — log in again"
+                continue
+            # Some providers do not reissue a refresh token; keep the one we have.
+            if not tokens.refresh_token:
+                tokens.refresh_token = refresh_token
+            self._store_session(record, tokens)
+            await self.store.put_account(record)
+            self._cache_session(record)
+            runtime = self.accounts.get(account_id)
+            if runtime is not None:
+                runtime.disabled_reason = None
+            logger.info("renewed subscription session for %r", account_id)
+
+    async def watch_oauth_sessions(self) -> None:
+        interval = max(5.0, self.config.oauth.refresh_interval_seconds)
+        while True:
+            await asyncio.sleep(interval)
+            try:
+                await self.refresh_oauth_sessions()
+            except Exception as exc:  # noqa: BLE001 - a blip must not kill routing
+                logger.warning("session refresh pass failed: %s", exc)
+
+    async def create_key(self, name: str, **options: Any) -> Tuple[str, Dict[str, Any]]:
+        """Mint a managed key. The plaintext is returned once and never stored."""
+        name = (name or "").strip()
+        if not name:
+            raise GatewayError(400, "a key needs a name", kind="invalid_request_error")
+        await self.refresh_keys()
+        if self.keys.by_name(name) is not None:
+            raise GatewayError(
+                409, f"a key named {name!r} already exists", kind="invalid_request_error"
+            )
+
+        pool = list(options.get("pool") or [])
+        unknown = [account_id for account_id in pool if account_id not in self.accounts]
+        if unknown:
+            raise GatewayError(
+                400,
+                "unknown account(s) in pool: {}".format(", ".join(sorted(unknown))),
+                kind="invalid_request_error",
+            )
+
+        plaintext = generate_key()
+        template = KeyConfig(
+            key="",
+            name=name,
+            models=list(options.get("models") or ["*"]),
+            pool=pool,
+            rpm=options.get("rpm"),
+            spend_cap_usd=options.get("spend_cap_usd"),
+            priority=str(options.get("priority") or "interactive"),
+            max_wait_seconds=options.get("max_wait_seconds"),
+            session_cap_usd=options.get("session_cap_usd"),
+            session_max_turns=options.get("session_max_turns"),
+            admin=bool(options.get("admin")),
+        )
+        record = record_from_config(template, plaintext)
+        await self.store.put_key(record)
+        await self.refresh_keys()
+        redacted = {k: v for k, v in record.items() if k != "key_hash"}
+        return plaintext, redacted
+
+    #: What a key's record may have changed after it is minted. The key material
+    #: itself is absent: a key is rotated by issuing a new one, not by editing a
+    #: hash in place, and `admin` is absent because quietly promoting a tenant key
+    #: is not an edit anybody should be able to make by typo.
+    KEY_EDITABLE = (
+        "models", "pool", "rpm", "spend_cap_usd", "priority", "max_wait_seconds",
+        "session_cap_usd", "session_max_turns",
+    )
+
+    async def update_key(self, name: str, **changes: Any) -> Dict[str, Any]:
+        """Edit a managed key's scope in place.
+
+        Without this, narrowing a key's pool means revoking it and reissuing — which
+        means every client holding it breaks. A key scoped to an account that has
+        since been removed is the common case, and it blocks config reload until
+        somebody fixes it.
+        """
+        if self.keys.is_config_key(name):
+            raise GatewayError(
+                409,
+                f"{name!r} is declared in the config file; edit it there and reload",
+                kind="invalid_request_error",
+            )
+        records = {str(r.get("name")): r for r in await self.store.list_keys()}
+        record = records.get(name)
+        if record is None:
+            raise GatewayError(404, f"no managed key {name!r}", kind="not_found_error")
+
+        rejected = [k for k in changes if k not in self.KEY_EDITABLE]
+        if rejected:
+            raise GatewayError(
+                400,
+                "cannot change {} on a key. Editable: {}".format(
+                    ", ".join(sorted(rejected)), ", ".join(self.KEY_EDITABLE)
+                ),
+                kind="invalid_request_error",
+            )
+        if changes.get("pool") is not None:
+            pool = list(changes["pool"])
+            unknown = [a for a in pool if a not in self.accounts]
+            if unknown:
+                raise GatewayError(
+                    400,
+                    "unknown account(s) in pool: {}".format(", ".join(sorted(unknown))),
+                    kind="invalid_request_error",
+                )
+            changes["pool"] = pool
+        if changes.get("priority") is not None:
+            priority = str(changes["priority"])
+            if priority not in ("interactive", "batch", "background"):
+                raise GatewayError(
+                    400,
+                    "priority must be interactive, batch or background",
+                    kind="invalid_request_error",
+                )
+            changes["priority"] = priority
+
+        # rpm, cap and wait are nullable: "no limit" has to be expressible, or a
+        # cap set once could never be lifted without reissuing the key.
+        nullable = ("rpm", "spend_cap_usd", "max_wait_seconds")
+        for field_name, value in changes.items():
+            if value is None and field_name not in nullable:
+                continue
+            record[field_name] = value
+        await self.store.put_key(record)
+        await self.refresh_keys()
+        return {k: v for k, v in record.items() if k != "key_hash"}
+
+    async def revoke_key(self, name: str) -> bool:
+        """Revoke a managed key. Config keys belong to the file, not to the API."""
+        if self.keys.is_config_key(name):
+            raise GatewayError(
+                409,
+                f"{name!r} is declared in the config file; remove it there and reload",
+                kind="invalid_request_error",
+            )
+        removed = await self.store.delete_key(name)
+        await self.refresh_keys()
+        return removed
+
+    def reload(self, config: Config) -> Dict[str, Any]:
+        """Swap in a new config without dropping in-flight state.
+
+        Accounts that survive the reload keep their limit mirror, breaker and
+        counters — a config edit must not hand every account a clean bill of health
+        and re-stampede an upstream that was cooling for a good reason.
+        """
+        before = set(self.accounts)
+        after = [a.id for a in config.accounts]
+
+        for account_config in config.accounts:
+            existing = self.accounts.get(account_config.id)
+            if existing is None:
+                self.accounts[account_config.id] = AccountRuntime(
+                    config=account_config,
+                    mirror=_mirror_for(account_config),
+                    breaker=CircuitBreaker(
+                        failure_threshold=config.breaker.failure_threshold,
+                        cooldown_seconds=config.breaker.cooldown_seconds,
+                    ),
+                )
+                self.upstreams[account_config.id] = self._make_upstream(account_config)
+                continue
+            credentials_changed = (
+                existing.config.api_key != account_config.api_key
+                or existing.config.base_url != account_config.base_url
+                or existing.config.type != account_config.type
+            )
+            existing.config = account_config
+            existing.mirror.observable = account_config.observable_limits
+            existing.mirror.assumed_headroom = account_config.assumed_headroom
+            existing.breaker.failure_threshold = config.breaker.failure_threshold
+            existing.breaker.cooldown_seconds = config.breaker.cooldown_seconds
+            if credentials_changed:
+                self.upstreams[account_config.id] = self._make_upstream(account_config)
+                # New credentials deserve a fresh chance: a rotated key is the usual
+                # reason an account was disabled in the first place.
+                existing.disabled_reason = None
+                existing.breaker.record_success()
+
+        # Only config accounts are governed by the file; managed ones live in the
+        # store and must survive a reload of it.
+        removed = [
+            account_id for account_id in before
+            if account_id not in after and account_id in self._config_account_ids
+        ]
+        for account_id in removed:
+            self.accounts.pop(account_id, None)
+            self.upstreams.pop(account_id, None)
+        self._config_account_ids = set(after)
+
+        managed = list(self.keys._managed)  # noqa: SLF001 - same object's own state
+        self.config = config
+        self.keys = KeyRegistry(config.keys)
+        self.keys.set_managed(managed)
+        self.router = Router(config.routing.strategy, config.routing.weights)
+        # Reconfigured, not rebuilt: what the estimator has learned about each model
+        # and what the advisor has counted are observed state, and a reload must not
+        # send the pool back to leasing ceilings for twenty requests of every model.
+        # Managed keys are preserved a few lines above for the same reason.
+        self.estimator.reconfigure(
+            mode=config.routing.output_estimate,
+            quantile=config.routing.output_estimate_quantile,
+            min_samples=config.routing.output_estimate_min_samples,
+            window=config.routing.output_estimate_window,
+            floor=config.routing.output_estimate_floor,
+            max_undershoot=config.routing.output_estimate_max_undershoot,
+        )
+        self.cache_advisor.reconfigure(config.cache.advice_min_requests)
+        self.pacing.reconfigure(**_pacing_settings(config))
+        self.gate.max_size = config.queue.max_size
+        self.gate.default_max_wait = config.queue.default_max_wait_seconds
+        self._file_pricing_table = config.pricing_table
+        # The file has just replaced every account's settings and the router. A
+        # console setting is an overlay on the file, not a one-time edit to it, so
+        # both go back on top — otherwise saving an unrelated line in the YAML
+        # would quietly revert them.
+        self._apply_all_overrides()
+        self.apply_settings()
+
+        added = [account_id for account_id in after if account_id not in before]
+        return {
+            "accounts": after,
+            "added": added,
+            "removed": removed,
+            "strategy": self.router.strategy,
+        }
+
+    def reload_from_path(self) -> Dict[str, Any]:
+        """Re-read the config file. A broken file leaves the running config alone."""
+        if not self.config.path:
+            raise GatewayError(
+                400, "gateway was not started from a config file", kind="invalid_request_error"
+            )
+        try:
+            fresh = Config.load(self.config.path)
+        except (OSError, ConfigError) as exc:
+            raise GatewayError(
+                400, f"config reload failed, keeping the running config: {exc}",
+                kind="invalid_request_error",
+            ) from exc
+        return self.reload(fresh)
+
+    async def watch_config(self) -> None:
+        """Poll the config file's mtime and reload when it moves."""
+        interval = self.config.server.hot_reload_seconds
+        if not self.config.path or interval <= 0:
+            return
+        try:
+            last = os.path.getmtime(self.config.path)
+        except OSError:
+            return
+        while True:
+            await asyncio.sleep(interval)
+            try:
+                current = os.path.getmtime(self.config.path)
+            except OSError:
+                continue
+            if current == last:
+                continue
+            last = current
+            try:
+                result = self.reload_from_path()
+            except GatewayError as exc:
+                logger.warning("config reload rejected: %s", exc.message)
+                continue
+            logger.info("config reloaded: %s", json.dumps(result))
+
+    async def aclose(self) -> None:
+        if self._client is not None and self._owns_client:
+            await self._client.aclose()
+            self._client = None
+        await self.store.close()
+
+    @property
+    def client(self) -> httpx.AsyncClient:
+        if self._client is None:
+            raise RuntimeError("gateway.startup() has not run")
+        return self._client
+
+    # ---- public API ----------------------------------------------------------
+
+    async def complete(
+        self,
+        body: Dict[str, Any],
+        headers: Mapping[str, str],
+        key: KeyConfig,
+        path: str = MESSAGES_PATH,
+    ) -> Completion:
+        """Non-streaming Messages request, with transparent failover."""
+        plan = await self._prepare(body, headers, key)
+        last_error: Optional[GatewayError] = None
+
+        while True:
+            try:
+                account, decision = await self._acquire(plan)
+            except _SpillToBatch:
+                return await self._spill(plan)
+            result, classification, latency = await self._dispatch(account, plan, path)
+            plan.event.attempts.append(
+                Attempt(
+                    account_id=account.id,
+                    status=result.status if result else None,
+                    kind=classification.kind,
+                    latency=round(latency, 4),
+                    detail=classification.detail[:200],
+                )
+            )
+            plan.event.decision = decision.to_dict()
+
+            if classification.kind == "ok" and result is not None:
+                usage = Usage.from_body(result.body)
+                await self._settle(account, plan, decision, usage, latency, result.status)
+                return Completion(
+                    status=result.status,
+                    headers=_response_headers(result.headers, account.id),
+                    content=result.raw,
+                    event=plan.event,
+                )
+
+            account.apply(classification, time.time(), plan.model)
+            if classification.request_action is RequestAction.RETURN and result is not None:
+                plan.event.status = result.status
+                plan.event.error = classification.kind
+                plan.event.latency = time.time() - plan.started
+                self.events.record(plan.event)
+                return Completion(
+                    status=result.status,
+                    headers=_response_headers(result.headers, account.id),
+                    content=result.raw,
+                    event=plan.event,
+                )
+
+            last_error = GatewayError(
+                502 if result is None else result.status,
+                classification.detail or classification.kind,
+                kind=classification.kind,
+            )
+            if not self._may_retry(plan, account, classification):
+                await self._fail(plan, last_error)
+                raise last_error
+            await self._backoff(plan)
+
+    async def stream(
+        self,
+        body: Dict[str, Any],
+        headers: Mapping[str, str],
+        key: KeyConfig,
+        path: str = MESSAGES_PATH,
+    ) -> Tuple[Dict[str, str], AsyncIterator[bytes]]:
+        """Streaming Messages request.
+
+        Failover is transparent right up until the first ``content_block_delta``
+        reaches the client. After that the client is already rendering, so a failure
+        can only be reported as an SSE error frame — that boundary is the one thing
+        the gateway cannot hide, and it is documented rather than papered over.
+        """
+        plan = await self._prepare(body, headers, key, streamed=True)
+        response_headers = {
+            "content-type": "text/event-stream",
+            "cache-control": "no-cache",
+            "connection": "keep-alive",
+        }
+        return response_headers, self._stream_iter(plan, path)
+
+    # ---- internals -----------------------------------------------------------
+
+    async def _prepare(
+        self,
+        body: Dict[str, Any],
+        headers: Mapping[str, str],
+        key: KeyConfig,
+        streamed: bool = False,
+    ) -> _Plan:
+        model = str(body.get("model") or "")
+        if not model:
+            raise GatewayError(400, "request is missing 'model'", kind="invalid_request_error")
+        if not key.supports_model(model):
+            raise GatewayError(
+                403,
+                f"key {key.name!r} is not allowed to use model {model!r}",
+                kind="permission_error",
+            )
+
+        now = time.time()
+        if key.rpm is not None:
+            count = await self.store.record_key_request(key.name, now)
+            if count > key.rpm:
+                raise GatewayError(
+                    429,
+                    f"key {key.name!r} is over its {key.rpm} rpm limit",
+                    kind="rate_limit_error",
+                    retry_after=60.0,
+                )
+        if key.spend_cap_usd is not None:
+            spent = await self.store.get_spend(
+                SCOPE_KEY, key.name, self.config.spend.window_seconds
+            )
+            if spent >= key.spend_cap_usd:
+                raise GatewayError(
+                    429,
+                    f"key {key.name!r} has spent ${spent:.2f} of its "
+                    f"${key.spend_cap_usd:.2f} cap in the last "
+                    f"{self.config.spend.window_hours:.0f}h",
+                    kind="rate_limit_error",
+                )
+
+        lowered = {k.lower(): v for k, v in headers.items()}
+        priority = parse_priority(
+            lowered.get(PRIORITY_HEADER), parse_priority(key.priority)
+        )
+        max_wait = _positive_float(
+            lowered.get(MAX_WAIT_HEADER),
+            key.max_wait_seconds
+            if key.max_wait_seconds is not None
+            else self.config.queue.default_max_wait_seconds,
+        )
+
+        requested_model = model
+        model, body = await self._downshift(model, body, key, priority)
+
+        # Diagnosis before dispatch: whether the caller asked for caching at all,
+        # and how much stable head a breakpoint would have covered. Booleans and
+        # token counts only — the body itself is never kept.
+        marked = has_cache_breakpoint(body)
+        prefix_tokens = cacheable_prefix_tokens(body)
+        if not marked and self.config.cache.auto_breakpoint:
+            body, inserted = insert_cache_breakpoint(body)
+            marked = inserted
+
+        session = compute_session_key(body, headers, scope=key.name)
+        await self._check_session_caps(key, session)
+        owner = await self.store.get_affinity(session)
+        estimate = estimate_request(
+            body,
+            self.config.routing.estimate_safety_margin,
+            predictor=self.estimator.predict,
+        )
+        request_id = "req_" + uuid.uuid4().hex[:20]
+
+        event = RequestEvent(
+            request_id=request_id,
+            started_at=now,
+            model=model,
+            key_name=key.name,
+            session_key=session,
+            streamed=streamed,
+        )
+        event.priority = priority_name(priority)
+        event.model_requested = requested_model if requested_model != model else ""
+        event.cache_breakpoint = marked
+        event.prefix_tokens = prefix_tokens
+        return _Plan(
+            body=body,
+            headers=dict(headers),
+            key=key,
+            model=model,
+            estimate=estimate,
+            session=session,
+            owner=owner,
+            event=event,
+            started=now,
+            deadline=now + self.config.retry.deadline_seconds,
+            priority=priority,
+            max_wait=max_wait,
+        )
+
+    async def _cached_pace(self) -> Optional[float]:
+        now = time.time()
+        if self._pace_cached_at + PACE_CACHE_SECONDS > now:
+            return self._pace_value
+        self._pace_value = await self._pace()
+        self._pace_cached_at = now
+        return self._pace_value
+
+    async def _ahead_of_pace(self) -> bool:
+        """True when enforcement should act. False whenever it cannot be sure."""
+        if not self.pacing.enforcing:
+            return False
+        pace = await self._cached_pace()
+        return pace is not None and pace > self.pacing.ahead_threshold
+
+    async def _downshift(
+        self, model: str, body: Dict[str, Any], key: KeyConfig, priority: int
+    ) -> Tuple[str, Dict[str, Any]]:
+        """Substitute a cheaper model for batch work while ahead of pace.
+
+        The one lever here that changes *what the caller gets* rather than when or
+        where they get it, so it is off unless configured, batch-priority only, and
+        recorded on the request. It also refuses to substitute a model the key is
+        not allowed to use: a pacing policy must not widen what a tenant can reach.
+        """
+        downshift = self.config.pacing.model_downshift
+        if not downshift or priority < PRIORITY_BATCH:
+            return model, body
+        if not await self._ahead_of_pace():
+            return model, body
+        substituted = map_model(model, downshift)
+        if substituted == model or not key.supports_model(substituted):
+            return model, body
+        updated = dict(body)
+        updated["model"] = substituted
+        return substituted, updated
+
+    async def _apply_pacing(self, plan: _Plan) -> None:
+        """Hold a batch request back while the pool is ahead of its quota pace.
+
+        Deliberately a wait rather than a rejection. Waiting is what spends a window
+        more slowly; rejecting just moves the same work to whenever the client
+        retries, which is usually immediately.
+        """
+        if not self.pacing.enforcing or plan.priority < PRIORITY_BATCH:
+            return
+        if not await self._ahead_of_pace():
+            return
+
+        # Cheaper than waiting, so it is tried first: the Batches API is priced
+        # below standard and spends a different upstream limit, which is the whole
+        # reason being ahead of pace is worth acting on at all.
+        if self.config.pacing.prefer_batch_lane_when_ahead and self._may_spill(plan):
+            raise _SpillToBatch()
+
+        delay = self.pacing.delay_for(
+            plan.priority, PRIORITY_BATCH, await self._cached_pace()
+        )
+        budget = min(
+            delay,
+            max(0.0, plan.deadline - time.time()),
+            max(0.0, plan.max_wait - plan.queued_seconds),
+        )
+        if budget <= 0:
+            return
+        started = time.time()
+        try:
+            await asyncio.sleep(budget)
+        finally:
+            waited = time.time() - started
+            plan.queued_seconds += waited
+            plan.event.paced_for = round(waited, 3)
+
+    async def _check_session_caps(self, key: KeyConfig, session: str) -> None:
+        """Bound one conversation, where `spend_cap_usd` bounds the whole key.
+
+        A key cap catches a tenant overspending across everything they do. Nothing
+        caught a single agent loop doing it inside that allowance, and one loop
+        resending a large prefix a few hundred times can be the whole cap with one
+        key's name on it — visible only afterwards, in a chart.
+        """
+        if not self.config.sessions.track:
+            return
+        window = self.config.spend.window_seconds
+
+        if key.session_max_turns is not None:
+            turns = await self.store.get_spend(SCOPE_SESSION_TURNS, session, window)
+            if turns >= key.session_max_turns:
+                raise GatewayError(
+                    429,
+                    f"this conversation has run {int(turns)} turns, at the "
+                    f"{key.session_max_turns}-turn limit for key {key.name!r}. Start "
+                    "a new conversation, or raise session_max_turns",
+                    kind="rate_limit_error",
+                )
+
+        if key.session_cap_usd is not None:
+            spent = await self.store.get_spend(SCOPE_SESSION, session, window)
+            if spent >= key.session_cap_usd:
+                raise GatewayError(
+                    429,
+                    f"this conversation has spent ${spent:.2f} of its "
+                    f"${key.session_cap_usd:.2f} limit for key {key.name!r} in the "
+                    f"last {self.config.spend.window_hours:.0f}h",
+                    kind="rate_limit_error",
+                )
+
+    async def sessions_report(self) -> Dict[str, Any]:
+        """The most expensive conversations in the spend window, worst first.
+
+        Sessions are unbounded in number, so this reports the tail that matters
+        rather than all of them: what a runaway looks like is one session far above
+        every other, and that is visible in twenty rows.
+        """
+        settings = self.config.sessions
+        if not settings.track:
+            return {"tracking": False, "sessions": []}
+
+        window = self.config.spend.window_seconds
+        spend = await self.store.spend_by_scope(SCOPE_SESSION, window)
+        turns = await self.store.spend_by_scope(SCOPE_SESSION_TURNS, window)
+
+        rows: List[Dict[str, Any]] = []
+        for session in set(spend) | set(turns):
+            session_turns = int(turns.get(session, 0.0))
+            session_spend = float(spend.get(session, 0.0))
+            reasons = []
+            if settings.runaway_turns and session_turns >= settings.runaway_turns:
+                reasons.append(f"{session_turns} turns")
+            if (
+                settings.runaway_spend_usd
+                and session_spend >= settings.runaway_spend_usd
+            ):
+                reasons.append(f"${session_spend:.2f}")
+            rows.append({
+                "session_key": session,
+                "turns": session_turns,
+                "spend_usd": round(session_spend, 6),
+                "runaway": bool(reasons),
+                "why": ", ".join(reasons),
+            })
+
+        rows.sort(key=lambda row: (row["spend_usd"], row["turns"]), reverse=True)
+        return {
+            "tracking": True,
+            "window_seconds": window,
+            "runaway_turns": settings.runaway_turns,
+            "runaway_spend_usd": settings.runaway_spend_usd,
+            "sessions": rows[: max(1, settings.report_limit)],
+        }
+
+    async def _acquire(self, plan: _Plan) -> Tuple[AccountRuntime, Decision]:
+        """Route, or wait on the queue for capacity that is known to be coming."""
+        await self._apply_pacing(plan)
+        while True:
+            now = time.time()
+            if now >= plan.deadline:
+                waited = self.config.retry.deadline_seconds
+                raise GatewayError(
+                    504,
+                    f"no capacity became available within {waited:.0f}s",
+                    kind="timeout_error",
+                )
+            ctx = RoutingContext(
+                model=plan.model,
+                estimate=plan.estimate,
+                session_key=plan.session,
+                now=now,
+                sticky_owner=plan.owner,
+                pool=plan.key.pool,
+                exclude=plan.excluded,
+            )
+            decision = self.router.select(self.accounts.values(), ctx)
+            if decision.chosen is not None:
+                plan.event.queued_for = round(plan.queued_seconds, 3)
+                return decision.chosen, decision
+
+            admission = admission_check(False, decision.soonest_available)
+            if admission.retry_after is None:
+                plan.event.decision = decision.to_dict()
+                error = GatewayError(
+                    503,
+                    admission.reason or "no eligible account",
+                    kind="overloaded_error",
+                )
+                await self._fail(plan, error)
+                raise error
+
+            if self._may_spill(plan):
+                raise _SpillToBatch()
+
+            remaining_wait = plan.max_wait - plan.queued_seconds
+            if remaining_wait <= 0:
+                error = GatewayError(
+                    429,
+                    f"waited {plan.max_wait:.0f}s for pool capacity without any becoming "
+                    "available",
+                    kind="rate_limit_error",
+                    retry_after=decision.soonest_available,
+                )
+                await self._fail(plan, error)
+                raise error
+
+            budget = min(
+                plan.deadline - now,
+                remaining_wait,
+                admission.retry_after + 0.25,
+            )
+            waited_from = time.time()
+            try:
+                await self.gate.wait(plan.priority, budget)
+            except QueueFull as exc:
+                error = GatewayError(
+                    429,
+                    str(exc),
+                    kind="rate_limit_error",
+                    retry_after=decision.soonest_available,
+                )
+                await self._fail(plan, error)
+                raise error from exc
+            finally:
+                plan.queued_seconds += time.time() - waited_from
+
+    def _may_spill(self, plan: _Plan) -> bool:
+        return (
+            self.config.batch.enabled
+            and plan.priority >= PRIORITY_BATCH
+            and not plan.event.streamed
+            and not plan.spilled
+            and self._batch_account(plan) is not None
+        )
+
+    def _batch_account(self, plan: _Plan) -> Optional[AccountRuntime]:
+        """Batches have their own upstream rate-limit pool, so a messages-saturated
+        account is still a fine place to send one. Prefer the cache owner."""
+        pool = set(plan.key.pool) if plan.key.pool else None
+        candidates = [
+            account
+            for account in self.accounts.values()
+            if (pool is None or account.id in pool)
+            and account.state(time.time()) is not AccountState.DISABLED
+            and account.config.supports_model(plan.model)
+            and plan.model not in account.unsupported_models
+        ]
+        if not candidates:
+            return None
+        for account in candidates:
+            if account.id == plan.owner:
+                return account
+        return candidates[0]
+
+    async def _spill(self, plan: _Plan) -> Completion:
+        """Run a saturated batch-priority request through the Batches API."""
+        plan.spilled = True
+        account = self._batch_account(plan)
+        if account is None:
+            raise GatewayError(503, "no account can accept a batch", kind="overloaded_error")
+
+        deadline = min(
+            plan.deadline, time.time() + max(0.0, plan.max_wait - plan.queued_seconds)
+        )
+        started = time.time()
+        try:
+            outcome = await batch_lane.run_single(
+                self.client,
+                self.upstreams[account.id],
+                plan.body,
+                plan.headers,
+                self.config.batch.poll_interval_seconds,
+                deadline,
+            )
+        except batch_lane.BatchTimeout as exc:
+            error = GatewayError(
+                429,
+                "batch did not finish within this request's wait budget; it was "
+                "cancelled upstream",
+                kind="rate_limit_error",
+                extra_headers=(
+                    {"x-tokenbiryani-batch-id": exc.batch_id} if exc.batch_id else None
+                ),
+            )
+            await self._fail(plan, error)
+            raise error from exc
+        except (batch_lane.BatchUnavailable, httpx.HTTPError) as exc:
+            # The spill lane is an optimisation, never a dependency: fall back to
+            # the normal queue rather than failing the request.
+            logger.info("batch spill unavailable, falling back to queue: %s", exc)
+            plan.event.attempts.append(
+                Attempt(account_id=account.id, kind="batch_unavailable", detail=str(exc)[:200])
+            )
+            return await self._resume_after_spill(plan)
+
+        latency = time.time() - started
+        usage = Usage.from_body(outcome.message)
+        plan.event.via = "batch"
+        plan.event.batch_id = outcome.batch_id
+        plan.event.attempts.append(
+            Attempt(
+                account_id=account.id,
+                status=200,
+                kind="batch",
+                latency=round(latency, 4),
+                detail=f"{outcome.polls} polls",
+            )
+        )
+        decision = Decision(account, [], "batch_spill")
+        await self._settle(
+            account, plan, decision, usage, latency, 200,
+            cost_multiplier=self.config.batch.cost_multiplier,
+        )
+        return Completion(
+            status=200,
+            headers={
+                "content-type": "application/json",
+                "x-tokenbiryani-account": account.id,
+                "x-tokenbiryani-via": "batch",
+                "x-tokenbiryani-batch-id": outcome.batch_id,
+            },
+            content=json.dumps(outcome.message).encode("utf-8"),
+            event=plan.event,
+        )
+
+    async def _resume_after_spill(self, plan: _Plan) -> Completion:
+        """Re-enter the normal path once, with spilling disabled for this request."""
+        account, decision = await self._acquire(plan)
+        result, classification, latency = await self._dispatch(account, plan, MESSAGES_PATH)
+        plan.event.attempts.append(
+            Attempt(
+                account_id=account.id,
+                status=result.status if result else None,
+                kind=classification.kind,
+                latency=round(latency, 4),
+                detail=classification.detail[:200],
+            )
+        )
+        plan.event.decision = decision.to_dict()
+        if classification.kind == "ok" and result is not None:
+            usage = Usage.from_body(result.body)
+            await self._settle(account, plan, decision, usage, latency, result.status)
+            return Completion(
+                status=result.status,
+                headers=_response_headers(result.headers, account.id),
+                content=result.raw,
+                event=plan.event,
+            )
+        account.apply(classification, time.time(), plan.model)
+        error = GatewayError(
+            502 if result is None else result.status,
+            classification.detail or classification.kind,
+            kind=classification.kind,
+        )
+        await self._fail(plan, error)
+        raise error
+
+    async def _dispatch(
+        self, account: AccountRuntime, plan: _Plan, path: str
+    ) -> Tuple[Any, Classification, float]:
+        """One non-streaming attempt, with the lease held for its duration."""
+        lease = account.mirror.reserve(plan.estimate, account.id)
+        account.inflight += 1
+        started = time.time()
+        result = None
+        try:
+            result = await self.upstreams[account.id].send(
+                self.client, path, plan.body, plan.headers
+            )
+            classification = classify(result.status, result.headers, result.body)
+        except Exception as exc:  # noqa: BLE001 - transport failures are data here
+            classification = classify_exception(exc)
+        latency = time.time() - started
+        now = time.time()
+        if result is not None:
+            account.observe_headers(dict(result.headers), now)
+        usage = Usage.from_body(result.body if result else None)
+        account.mirror.release(
+            lease,
+            actual_input=usage.billed_input or None,
+            actual_output=usage.output_tokens or None,
+            now=now,
+        )
+        account.inflight -= 1
+        await self.gate.wake_one()
+        return result, classification, latency
+
+    async def _stream_iter(self, plan: _Plan, path: str) -> AsyncIterator[bytes]:
+        while True:
+            try:
+                account, decision = await self._acquire(plan)
+            except GatewayError as exc:
+                yield sse.error_event(exc.message, exc.kind)
+                return
+
+            plan.event.decision = decision.to_dict()
+            upstream = self.upstreams[account.id]
+            lease = account.mirror.reserve(plan.estimate, account.id)
+            account.inflight += 1
+            started = time.time()
+            collector = sse.UsageCollector()
+            committed = False
+            classification: Optional[Classification] = None
+            status: Optional[int] = None
+
+            try:
+                context = upstream.open_stream(self.client, path, plan.body, plan.headers)
+                response = await context.__aenter__()
+                try:
+                    status = int(response.status_code)
+                    if status != 200:
+                        await response.aread()
+                        body = _safe_json(response)
+                        classification = classify(status, dict(response.headers), body)
+                        account.observe_headers(dict(response.headers), time.time())
+                    else:
+                        account.observe_headers(dict(response.headers), time.time())
+                        buffered: List[bytes] = []
+                        grace = started + self.config.retry.first_token_grace_seconds
+                        async for chunk in upstream.iter_sse(response):
+                            collector.feed(chunk)
+                            if not committed:
+                                buffered.append(chunk)
+                                seen_content = sse.FIRST_CONTENT_EVENT in b"".join(buffered)
+                                if seen_content or time.time() >= grace:
+                                    # Point of no return: flush and commit.
+                                    committed = True
+                                    if plan.event.ttft is None:
+                                        plan.event.ttft = round(time.time() - started, 4)
+                                    for pending in buffered:
+                                        yield pending
+                                    buffered = []
+                            else:
+                                yield chunk
+                        if not committed:
+                            # Stream ended before any delta — a complete, short response.
+                            committed = True
+                            if plan.event.ttft is None:
+                                plan.event.ttft = round(time.time() - started, 4)
+                            for pending in buffered:
+                                yield pending
+                        classification = classify(200)
+                finally:
+                    await context.__aexit__(None, None, None)
+            except Exception as exc:  # noqa: BLE001
+                classification = classify_exception(exc)
+
+            collector.finish()
+            latency = time.time() - started
+            now = time.time()
+            usage = Usage.from_body(collector.as_body())
+            account.mirror.release(
+                lease,
+                actual_input=usage.billed_input or None,
+                actual_output=usage.output_tokens or None,
+                now=now,
+            )
+            account.inflight -= 1
+            await self.gate.wake_one()
+
+            assert classification is not None
+            plan.event.attempts.append(
+                Attempt(
+                    account_id=account.id,
+                    status=status,
+                    kind=classification.kind,
+                    latency=round(latency, 4),
+                    detail=classification.detail[:200],
+                )
+            )
+
+            if classification.kind == "ok":
+                await self._settle(account, plan, decision, usage, latency, status or 200)
+                return
+
+            account.apply(classification, now, plan.model)
+            if committed:
+                # Past the commit boundary: the client has content. Report, don't retry.
+                plan.event.error = classification.kind
+                plan.event.status = status
+                plan.event.latency = time.time() - plan.started
+                self.events.record(plan.event)
+                yield sse.error_event(
+                    classification.detail or classification.kind, classification.kind
+                )
+                return
+
+            if (
+                classification.request_action is RequestAction.RETURN
+                or not self._may_retry(plan, account, classification)
+            ):
+                message = classification.detail or classification.kind
+                await self._fail(plan, GatewayError(status or 502, message, classification.kind))
+                yield sse.error_event(message, classification.kind)
+                return
+
+            await self._backoff(plan)
+
+    def _may_retry(
+        self, plan: _Plan, account: AccountRuntime, classification: Classification
+    ) -> bool:
+        if not classification.retryable:
+            return False
+        if classification.request_action is RequestAction.RETRY_OTHER:
+            if account.id not in plan.excluded:
+                plan.excluded.append(account.id)
+        plan.attempts += 1
+        retry = self.config.retry
+        if plan.attempts >= retry.max_attempts:
+            return False
+        if len(plan.excluded) >= min(retry.max_accounts, len(self.accounts)):
+            return False
+        if time.time() >= plan.deadline:
+            return False
+        return True
+
+    async def _backoff(self, plan: _Plan) -> None:
+        retry = self.config.retry
+        delay = min(
+            retry.backoff_max_seconds,
+            retry.backoff_base_seconds * (2 ** max(0, plan.attempts - 1)),
+        )
+        await asyncio.sleep(random.uniform(0, delay))
+
+    async def _settle(
+        self,
+        account: AccountRuntime,
+        plan: _Plan,
+        decision: Decision,
+        usage: Usage,
+        latency: float,
+        status: int,
+        cost_multiplier: float = 1.0,
+    ) -> None:
+        """Record a success: stats, affinity, spend, and the event."""
+        now = time.time()
+        # What the model really returned, against what its lease reserved. The lease
+        # is the number that decided whether this account looked full.
+        self.estimator.observe(plan.model, plan.estimate.output_tokens, usage.output_tokens)
+        self.cache_advisor.observe(
+            plan.key.name,
+            plan.model,
+            plan.event.cache_breakpoint,
+            plan.event.prefix_tokens,
+            usage.input_tokens,
+            usage.cache_read_tokens,
+            usage.cache_creation_tokens,
+        )
+        price = self.config.price_for(plan.model)
+        cost = account.record_success(latency, usage, price, now, cost_multiplier)
+
+        await self.store.set_affinity(
+            plan.session, account.id, self.config.routing.affinity_ttl_seconds
+        )
+
+        if cost is not None:
+            await self.store.add_spend(SCOPE_KEY, plan.key.name, cost)
+            await self.store.add_spend(SCOPE_ACCOUNT, account.id, cost)
+
+        if self.config.sessions.track:
+            # A turn is recorded whether or not the model was priced: a runaway loop
+            # on an unpriced model is still a runaway loop, and a turn cap that only
+            # worked for priced models would fail exactly where the cost is unknown.
+            await self.store.add_spend(SCOPE_SESSION_TURNS, plan.session, 1.0)
+            if cost is not None:
+                await self.store.add_spend(SCOPE_SESSION, plan.session, cost)
+
+        saved = None
+        if price is not None and usage.cache_read_tokens:
+            saved = usage.cache_read_tokens * (price.input - price.cache_read) / 1_000_000.0
+
+        event = plan.event
+        event.account_id = account.id
+        event.status = status
+        event.latency = round(now - plan.started, 4)
+        event.input_tokens = usage.input_tokens
+        event.output_tokens = usage.output_tokens
+        event.cache_read_tokens = usage.cache_read_tokens
+        event.cache_creation_tokens = usage.cache_creation_tokens
+        event.cost_usd = None if cost is None else round(cost, 6)
+        event.saved_usd = None if saved is None else round(saved, 6)
+        event.affinity_broken = decision.affinity_broken
+        event.affinity_honored = decision.affinity_honored
+        self.events.record(event)
+        await self._record_usage(event)
+
+
+    async def _record_usage(self, event: RequestEvent) -> None:
+        """Persist one request's accounting so the charts survive a restart.
+
+        Never allowed to fail a request. The spend ledger is the billing record and
+        is written separately; this is the history a chart is drawn from, and a
+        dropped row costs one notch on a graph.
+        """
+        try:
+            await self.store.record_usage(sample_from_event(event))
+        except Exception:  # noqa: BLE001 - telemetry must not break the response
+            logger.warning("could not record usage for %s", event.request_id, exc_info=True)
+
+    async def _fail(self, plan: _Plan, error: GatewayError) -> None:
+        plan.event.status = error.status
+        plan.event.error = error.kind
+        plan.event.latency = round(time.time() - plan.started, 4)
+        self.events.record(plan.event)
+        # Failures are history too. A usage chart that shows only successes hides
+        # the outage, which is exactly what someone opens it to find.
+        await self._record_usage(plan.event)
+
+    async def simple_request(
+        self,
+        method: str,
+        path: str,
+        key: KeyConfig,
+        headers: Mapping[str, str],
+        body: Optional[Dict[str, Any]] = None,
+    ) -> Completion:
+        """Auxiliary endpoints (models, count_tokens): no leases, one failover hop.
+
+        These cost no quota, so they skip admission and scoring entirely — running
+        them through the router would be ceremony without a purpose.
+        """
+        now = time.time()
+        pool = set(key.pool) if key.pool else None
+        candidates = [
+            a
+            for a in self.accounts.values()
+            if (pool is None or a.id in pool) and a.state(now) is not AccountState.DISABLED
+        ]
+        if not candidates:
+            raise GatewayError(503, "no account available", kind="overloaded_error")
+
+        last: Optional[GatewayError] = None
+        for account in candidates[: self.config.retry.max_accounts]:
+            upstream = self.upstreams[account.id]
+            try:
+                if method.upper() == "GET":
+                    response = await self.client.get(
+                        upstream.url(path), headers=upstream.request_headers(headers)
+                    )
+                    result = type(
+                        "R", (), {"status": response.status_code, "headers": dict(response.headers),
+                                  "raw": response.content}
+                    )()
+                else:
+                    sent = await upstream.send(self.client, path, body or {}, headers)
+                    result = sent
+                account.observe_headers(dict(result.headers), time.time())
+                if result.status < 500 and result.status != 429:
+                    return Completion(
+                        status=result.status,
+                        headers=_response_headers(result.headers, account.id),
+                        content=result.raw,
+                        event=RequestEvent(request_id="aux", started_at=now),
+                    )
+                last = GatewayError(result.status, f"upstream returned {result.status}")
+            except Exception as exc:  # noqa: BLE001
+                last = GatewayError(502, f"{type(exc).__name__}: {exc}")
+        raise last or GatewayError(502, "all accounts failed")
+
+    # ---- introspection -------------------------------------------------------
+
+    async def pace_readings(self) -> List[PaceReading]:
+        """One reading per account that reports a window, plus the pool's budget.
+
+        An account that reports unified utilisation is paced on what it says about
+        itself. An API-key account says nothing about a week, so it can only be
+        paced against a budget the operator stated — and if none is stated it is
+        left out rather than guessed at.
+        """
+        now = time.time()
+        readings: List[PaceReading] = []
+        if not self.pacing.enabled:
+            return readings
+
+        for account in self.accounts.values():
+            window = account.mirror.unified.get(self.pacing.window)
+            if window is None or window.utilization is None:
+                continue
+            reading = self.pacing.from_unified(
+                account.id, window.utilization, window.seconds_to_reset(now), now
+            )
+            if reading is not None:
+                readings.append(reading)
+
+        if self.pacing.weekly_budget_usd:
+            since = max(1.0, now - _pacing_week_start(now))
+            spent = sum(
+                (await self.store.spend_by_scope(SCOPE_ACCOUNT, since)).values()
+            )
+            reading = self.pacing.from_budget("pool", spent, now)
+            if reading is not None:
+                readings.append(reading)
+        return readings
+
+    async def pacing_report(self) -> Dict[str, Any]:
+        return self.pacing.report(await self.pace_readings(), time.time())
+
+    async def _pace(self) -> Optional[float]:
+        """The tightest pace across every scope, or None when nothing is paced.
+
+        The tightest and not the mean: one account about to run dry on Wednesday is
+        the fact worth acting on, and averaging it against three healthy ones is how
+        that fact gets lost.
+        """
+        readings = await self.pace_readings()
+        if not readings:
+            return None
+        return max(reading.pace for reading in readings)
+
+    def capacity_horizon(self, minutes: int = 60, buckets: int = 12) -> Dict[str, Any]:
+        """Projected input-token capacity over the next hour.
+
+        Drawable only because we mirror per-account reset timestamps; a round-robin
+        proxy has no idea when anything refills.
+        """
+        now = time.time()
+        step = (minutes * 60.0) / buckets
+        series: List[Dict[str, Any]] = []
+        for index in range(buckets):
+            at = now + index * step
+            total = 0
+            for account in self.accounts.values():
+                if account.state(now) is AccountState.DISABLED:
+                    continue
+                if not account.mirror.observable:
+                    # Nothing is known about this account's future capacity, and a
+                    # horizon that quietly includes a guess is worse than one that
+                    # leaves it out.
+                    continue
+                window = account.mirror.input_tokens
+                if window.limit is None:
+                    continue
+                if window.reset_at is not None and at >= window.reset_at:
+                    total += window.limit
+                elif window.remaining is not None:
+                    total += max(0, window.remaining - account.mirror.reserved_input)
+            series.append({"offset_seconds": round(index * step), "input_tokens": total})
+        return {"minutes": minutes, "series": series}
+
+    def _identify(self, runtime: AccountRuntime, entry: Dict[str, Any]) -> Dict[str, Any]:
+        """Stamp the fields the console needs to *manage* an account, not just watch it.
+
+        `AccountRuntime.snapshot` is the live picture — states, meters, counters. None
+        of it says what the account is called, whether it is editable, or what it was
+        configured with, and a UI that offers a Rename button needs all three.
+        """
+        config = runtime.config
+        entry["name"] = getattr(config, "name", None) or runtime.id
+        entry["source"] = "config" if self.is_config_account(runtime.id) else "managed"
+        entry["enabled"] = bool(getattr(config, "enabled", True))
+        entry["base_url"] = getattr(config, "base_url", "") or ""
+        entry["models"] = list(getattr(config, "models", ["*"]))
+        entry["max_concurrency"] = getattr(config, "max_concurrency", None)
+        entry["spend_cap_usd"] = getattr(config, "spend_cap_usd", None)
+        entry["observable_limits"] = bool(getattr(config, "observable_limits", True))
+        override = self._account_overrides.get(runtime.id) or {}
+        entry["overridden"] = sorted(override) if override else []
+        if config.type == "oauth":
+            upstream = self.upstreams.get(runtime.id)
+            describe = getattr(upstream, "describe_source", None)
+            expiry = getattr(upstream, "source_expiry", None)
+            entry["token_source"] = getattr(upstream, "source_kind", "login")
+            entry["token_source_detail"] = describe() if describe is not None else ""
+            entry["token_expires_at"] = expiry() if expiry is not None else None
+        if config.type == "oauth":
+            entry.update(self._oauth_meta.get(runtime.id, {
+                "session_expires_at": None, "can_refresh": False,
+            }))
+        return entry
+
+    def account_detail(self, account_id: str) -> Optional[Dict[str, Any]]:
+        """One account, live picture plus the config the console can edit."""
+        runtime = self.accounts.get(account_id)
+        if runtime is None:
+            return None
+        payload = self._identify(runtime, runtime.snapshot(time.time()))
+        payload["recent_requests"] = self.events.recent_for(account_id, 25)
+        return payload
+
+    async def usage(
+        self,
+        window: Optional[str] = None,
+        bucket: Optional[str] = None,
+        group_by: str = "account",
+        account_id: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Bucketed usage history for the console's charts.
+
+        Unlike `snapshot`, this reads the store rather than memory, so it answers
+        for time the process was not running.
+        """
+        window_seconds = resolve_window(window)
+        bucket_seconds = resolve_bucket(window_seconds, bucket)
+        now = time.time()
+        # Ask for the whole bucket grid the aggregator will emit, not just the
+        # trailing window, or the oldest bucket comes back half empty.
+        end = (int(now) // bucket_seconds) * bucket_seconds + bucket_seconds
+        count = max(1, window_seconds // bucket_seconds)
+        rows = await self.store.usage_rows(end - count * bucket_seconds, end, account_id)
+        result = aggregate(rows, now, window_seconds, bucket_seconds, group_by)
+        result["account_id"] = account_id
+        # Names, so a chart legend can read "Work account" rather than "acct-02".
+        result["names"] = {
+            runtime.id: getattr(runtime.config, "name", None) or runtime.id
+            for runtime in self.accounts.values()
+        }
+        return result
+
+    def snapshot(self) -> Dict[str, Any]:
+        now = time.time()
+        accounts = [
+            self._identify(runtime, runtime.snapshot(now))
+            for runtime in self.accounts.values()
+        ]
+        ready = [
+            a
+            for a in self.accounts.values()
+            if a.state(now) is AccountState.READY
+        ]
+        ready_tokens = 0
+        # Accounts whose budget is a percentage cannot be added to a token count.
+        # Counting them as zero would read as "nothing available" on a pool that is
+        # in fact half full, so they are reported separately as headroom.
+        ready_headroom = None
+        for account in ready:
+            available = account.mirror.input_tokens.available(
+                account.mirror.reserved_input, now
+            )
+            if available is not None:
+                ready_tokens += available
+            elif account.mirror.unified_known:
+                headroom = account.mirror.headroom(now)
+                if ready_headroom is None or headroom > ready_headroom:
+                    ready_headroom = headroom
+        resets = []
+        for account in self.accounts.values():
+            reset = account.mirror.next_reset(now)
+            if reset is not None:
+                resets.append(reset)
+        return {
+            "strategy": self.router.strategy,
+            # What the console needs to explain an empty cost column: whether any
+            # model is priced at all, and — if the bundled table is in use — how old
+            # it is. A stale price should be visible, not silent.
+            "pricing": {
+                "models": len(self.config.pricing),
+                "as_of": self.config.pricing_as_of,
+                "source": self.config.pricing_source,
+            },
+            "accounts": accounts,
+            "pool": {
+                "total": len(self.accounts),
+                "ready": len(ready),
+                "cooling": sum(
+                    1 for a in self.accounts.values() if a.state(now) is AccountState.COOLING
+                ),
+                "disabled": sum(
+                    1 for a in self.accounts.values() if a.state(now) is AccountState.DISABLED
+                ),
+                "ready_input_tokens": ready_tokens,
+                #: Best headroom among ready accounts that report a fraction rather
+                #: than a count. None when every ready account reports a count.
+                "ready_headroom": (
+                    None if ready_headroom is None else round(ready_headroom, 4)
+                ),
+                "next_reset_seconds": round(min(resets), 1) if resets else None,
+            },
+            "queue": self.gate.snapshot(),
+            "stats": self.events.stats(),
+        }
+
+
+@dataclass
+class _Plan:
+    body: Dict[str, Any]
+    headers: Dict[str, str]
+    key: KeyConfig
+    model: str
+    estimate: TokenEstimate
+    session: str
+    owner: Optional[str]
+    event: RequestEvent
+    started: float
+    deadline: float
+    priority: int = 0
+    max_wait: float = 60.0
+    spilled: bool = False
+    queued_seconds: float = 0.0
+    excluded: List[str] = None  # type: ignore[assignment]
+    attempts: int = 0
+
+    def __post_init__(self) -> None:
+        if self.excluded is None:
+            self.excluded = []
+
+
+def _mirror_for(config: AccountConfig) -> LimitMirror:
+    return LimitMirror(
+        observable=config.observable_limits, assumed_headroom=config.assumed_headroom
+    )
+
+
+def _positive_float(raw: Optional[str], fallback: float) -> float:
+    """A client may shorten its own wait budget, never extend it past the operator's."""
+    if raw is None:
+        return fallback
+    try:
+        requested = float(raw)
+    except (TypeError, ValueError):
+        return fallback
+    if requested <= 0:
+        return 0.0
+    return min(requested, fallback)
+
+
+def _safe_json(response: httpx.Response) -> Optional[Dict[str, Any]]:
+    try:
+        parsed = response.json()
+    except ValueError:
+        return None
+    return parsed if isinstance(parsed, dict) else None