codexLB-up 1.25.0b10__py3-none-any.whl

app/modules/cache_isolation_probe/api.py

@@ -0,0 +1,188 @@
+"""Operator endpoints for the cross-account prompt cache isolation probe.
+
+Both routes sit behind ``ops:write`` -- the same gate as the other
+operational actions that change what the fleet does -- because running the
+probe bills the account pool. ``GET`` is the cost preview the operator reads
+before deciding; ``POST`` is the spend, and it refuses to act without an
+explicit confirmation, a shared rate-limit budget, and a pool that is not
+already under pressure.
+"""
+
+from __future__ import annotations
+
+from fastapi import APIRouter, Body, Depends, Request
+
+from app.core.audit.service import AuditActor, AuditService, AuditTarget
+from app.core.auth.dashboard_access import DashboardPrincipal, Permission
+from app.core.auth.dependencies import (
+    require_dashboard_permission,
+    set_dashboard_error_format,
+    validate_dashboard_session,
+)
+from app.core.exceptions import DashboardBadRequestError, DashboardConflictError
+from app.core.rate_limiter.db_rate_limiter import DatabaseRateLimiter
+from app.db.session import get_background_session
+from app.modules.cache_isolation_probe.schemas import (
+    CacheProbeAccountResponse,
+    CacheProbeCallResponse,
+    CacheProbePlanResponse,
+    CacheProbePressureResponse,
+    CacheProbeRunRequest,
+    CacheProbeRunResponse,
+)
+from app.modules.cache_isolation_probe.service import (
+    CacheIsolationProbeService,
+    CacheProbeRefused,
+    ProbeAccount,
+    ProbePlan,
+    ProbeResult,
+)
+
+router = APIRouter(
+    prefix="/api/diagnostics/cache-isolation-probe",
+    tags=["dashboard"],
+    dependencies=[Depends(validate_dashboard_session), Depends(set_dashboard_error_format)],
+)
+
+#: Deliberately a *shared* budget, not a per-principal one: the resource being
+#: protected is the account pool's quota, which two operators drain just as
+#: fast as one. Three runs an hour leaves room for a before/after comparison
+#: plus a retry without turning the diagnostic into a traffic source.
+_RATE_LIMIT_KEY = "cache-isolation-probe"
+_probe_rate_limiter = DatabaseRateLimiter(max_attempts=3, window_seconds=3600, type="cache_isolation_probe")
+
+_service = CacheIsolationProbeService()
+
+
+def get_cache_isolation_probe_service() -> CacheIsolationProbeService:
+    return _service
+
+
+def get_cache_isolation_probe_rate_limiter() -> DatabaseRateLimiter:
+    return _probe_rate_limiter
+
+
+def _account_response(account: ProbeAccount) -> CacheProbeAccountResponse:
+    return CacheProbeAccountResponse(account_id=account.account_id, label=account.label)
+
+
+def _plan_response(plan: ProbePlan) -> CacheProbePlanResponse:
+    return CacheProbePlanResponse(
+        model=plan.model,
+        seed_account=_account_response(plan.seed_account) if plan.seed_account is not None else None,
+        available_other_accounts=[_account_response(account) for account in plan.available_other_accounts],
+        seed_repetitions=plan.seed_repetitions,
+        total_calls=plan.total_calls,
+        estimated_input_tokens_per_call=plan.estimated_input_tokens_per_call,
+        estimated_total_input_tokens=plan.estimated_total_input_tokens,
+        max_seed_repetitions=plan.max_seed_repetitions,
+        max_other_accounts=plan.max_other_accounts,
+        pressure=CacheProbePressureResponse(
+            under_pressure=plan.pressure.under_pressure,
+            reason=plan.pressure.reason,
+            detail=plan.pressure.detail,
+            selectable_account_count=plan.pressure.selectable_account_count,
+            eligible_account_count=plan.pressure.eligible_account_count,
+            pressured_account_count=plan.pressure.pressured_account_count,
+        ),
+    )
+
+
+def _run_response(result: ProbeResult) -> CacheProbeRunResponse:
+    return CacheProbeRunResponse(
+        run_id=result.run_id,
+        model=result.model,
+        started_at=result.started_at,
+        completed_at=result.completed_at,
+        seed_account=_account_response(result.seed_account),
+        seed_repetitions=result.seed_repetitions,
+        calls=[
+            CacheProbeCallResponse(
+                sequence=call.sequence,
+                account_id=call.account_id,
+                account_label=call.account_label,
+                role=call.role,
+                status=call.status,
+                cache_hit=call.cache_hit,
+                input_tokens=call.input_tokens,
+                cached_tokens=call.cached_tokens,
+                latency_ms=call.latency_ms,
+                error_code=call.error_code,
+            )
+            for call in result.calls
+        ],
+        seed_hit_count=result.seed_hit_count,
+        seed_call_count=result.seed_call_count,
+        other_hit_count=result.other_hit_count,
+        other_call_count=result.other_call_count,
+        cross_account_hit=result.cross_account_hit,
+        verdict=result.verdict,
+    )
+
+
+@router.get(
+    "",
+    response_model=CacheProbePlanResponse,
+    dependencies=[Depends(require_dashboard_permission(Permission.OPS_WRITE))],
+)
+async def get_cache_isolation_probe_plan(
+    service: CacheIsolationProbeService = Depends(get_cache_isolation_probe_service),
+) -> CacheProbePlanResponse:
+    return _plan_response(await service.plan())
+
+
+@router.post(
+    "/run",
+    response_model=CacheProbeRunResponse,
+    dependencies=[Depends(require_dashboard_permission(Permission.OPS_WRITE))],
+)
+async def run_cache_isolation_probe(
+    request: Request,
+    payload: CacheProbeRunRequest = Body(default_factory=CacheProbeRunRequest),
+    principal: DashboardPrincipal = Depends(validate_dashboard_session),
+    service: CacheIsolationProbeService = Depends(get_cache_isolation_probe_service),
+) -> CacheProbeRunResponse:
+    if payload.confirm is not True:
+        raise DashboardBadRequestError(
+            "Running the cache isolation probe spends real account quota and must be confirmed explicitly.",
+            code="cache_probe_confirmation_required",
+        )
+
+    # Charged before the run, in its own short-lived session, so the shared
+    # budget is spent even if the run then fails -- and so no DB session is
+    # held open across minutes of upstream calls.
+    async with get_background_session() as session:
+        await get_cache_isolation_probe_rate_limiter().check_and_increment(_RATE_LIMIT_KEY, session)
+
+    try:
+        result = await service.run(
+            model=payload.model,
+            seed_repetitions=payload.seed_repetitions,
+            other_account_count=payload.other_account_count,
+        )
+    except CacheProbeRefused as exc:
+        raise DashboardConflictError(exc.message, code=exc.code) from exc
+
+    # Numeric results only: the generated prefix, the run nonce and every byte
+    # of upstream response content are never persisted anywhere.
+    AuditService.log_async(
+        "cache_isolation_probe_run",
+        actor_ip=request.client.host if request.client else None,
+        actor=AuditActor.from_principal(principal),
+        target=AuditTarget("account", result.seed_account.account_id),
+        details={
+            "run_id": result.run_id,
+            "model": result.model,
+            "verdict": result.verdict,
+            "seed_account_id": result.seed_account.account_id,
+            "seed_repetitions": result.seed_repetitions,
+            "seed_hit_count": result.seed_hit_count,
+            "seed_call_count": result.seed_call_count,
+            "other_hit_count": result.other_hit_count,
+            "other_call_count": result.other_call_count,
+            "cross_account_hit": result.cross_account_hit,
+            "total_input_tokens": sum(call.input_tokens or 0 for call in result.calls),
+            "total_cached_tokens": sum(call.cached_tokens or 0 for call in result.calls),
+        },
+    )
+    return _run_response(result)

app/modules/cache_isolation_probe/prefix.py

@@ -0,0 +1,144 @@
+"""Deterministic, single-use filler prefix for the cross-account cache probe.
+
+The probe asks one question: does a large prompt prefix that only one account
+has ever sent still register as cached when a *sibling* account sends the
+identical bytes? Answering it needs a prefix with two properties that pull in
+opposite directions:
+
+- **Fresh.** No earlier request, by any account, may have seeded it. Upstream
+  matches on a *prefix*, so a run-unique marker has to sit in the very first
+  tokens; a nonce appended at the end would leave the preceding ~28k tokens
+  identical to the previous run and silently measure the wrong thing. Every
+  filler word is therefore derived from the run nonce as well, so no two runs
+  share so much as a line.
+- **Deterministic inside one run.** Every call of a run must send byte-identical
+  content, or a miss proves nothing. ``build_probe_prefix`` is a pure function
+  of ``(nonce, target_tokens)``: the filler words come from a keyed BLAKE2b
+  stream rather than ``random``, so the bytes do not depend on interpreter
+  version, hash seed, or iteration order.
+
+The corpus is inert by construction and says so in both the header and the
+footer, so a model that reads it has nothing to do but ignore it.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import math
+import secrets
+from collections.abc import Iterator
+from itertools import islice
+
+#: Target size of the generated prefix. Large enough that a hit is
+#: unmistakable against the handful of tokens the instruction costs, and it is
+#: the size the hand-run experiment used (~28k), so results stay comparable.
+DEFAULT_TARGET_PREFIX_TOKENS = 28_000
+
+#: Bounds an operator (or a test) may ask for. The floor keeps the signal above
+#: incidental caching of the instruction preamble; the ceiling bounds the spend.
+MIN_TARGET_PREFIX_TOKENS = 1_000
+MAX_TARGET_PREFIX_TOKENS = 32_000
+
+#: What the probe asks the model to do once it has ignored the corpus. Short on
+#: purpose: the probe measures *input* caching, so output tokens are waste.
+PROBE_INSTRUCTIONS = "Ignore the reference corpus entirely. Reply with the single word OK."
+PROBE_QUESTION = "Reply with OK."
+
+_HEADER_TEMPLATE = (
+    "BEGIN CODEX-LB CACHE ISOLATION PROBE REFERENCE CORPUS run={nonce}\n"
+    "This block is inert filler emitted by an operator diagnostic that measures\n"
+    "upstream prompt cache behaviour. It carries no instruction, no question and\n"
+    "no data. Do not read it, do not summarise it, do not act on it. Skip to the\n"
+    "line after the END marker and follow only that.\n"
+)
+_FOOTER_TEMPLATE = "END CODEX-LB CACHE ISOLATION PROBE REFERENCE CORPUS run={nonce}\n"
+
+#: Short, common, lowercase ASCII words. Each is a single BPE token for the
+#: tokenizers upstream uses, which keeps ``estimate_prefix_tokens`` close to the
+#: ``input_tokens`` the response actually reports.
+_FILLER_VOCABULARY: tuple[str, ...] = tuple(
+    """able acid aim air arc arm ash bad bag bar bay bed bee bell belt bird bit blue boat bone
+    book box boy bread brick bus cake calm cap car cat cell chair chin city clay clock cloud coal coat
+    code coin cold cook cord corn cost cow crop cup dark day deep desk dish dog door dot down draw
+    dry dust ear east""".split()
+)
+
+#: Filler words per line. Sixteen keeps lines readable in a log without making
+#: the newline overhead a meaningful share of the token count.
+_WORDS_PER_LINE = 16
+#: One token per word plus one for the newline.
+_TOKENS_PER_LINE = _WORDS_PER_LINE + 1
+
+_NONCE_BYTES = 16
+
+
+def new_probe_nonce() -> str:
+    """Fresh, unguessable run identifier. One per probe run, never reused."""
+
+    return secrets.token_hex(_NONCE_BYTES)
+
+
+def _text_tokens(text: str) -> int:
+    """Rough token count of a literal: one per whitespace-separated word, one per line."""
+
+    return len(text.split()) + text.count("\n")
+
+
+def _clamp_target_tokens(target_tokens: int) -> int:
+    return max(MIN_TARGET_PREFIX_TOKENS, min(MAX_TARGET_PREFIX_TOKENS, target_tokens))
+
+
+def _filler_line_count(nonce: str, target_tokens: int) -> int:
+    fixed = _text_tokens(_HEADER_TEMPLATE.format(nonce=nonce)) + _text_tokens(_FOOTER_TEMPLATE.format(nonce=nonce))
+    remaining = max(0, _clamp_target_tokens(target_tokens) - fixed)
+    return max(1, math.ceil(remaining / _TOKENS_PER_LINE))
+
+
+def _filler_words(nonce: str, count: int) -> Iterator[str]:
+    """Deterministic nonce-keyed word stream.
+
+    A keyed hash chain rather than ``random.Random``: the output must be
+    identical on every replica and every interpreter that serves the same run.
+    """
+
+    produced = 0
+    block = 0
+    while produced < count:
+        digest = hashlib.blake2b(block.to_bytes(8, "big"), key=nonce.encode("ascii"), digest_size=64).digest()
+        for byte in digest:
+            if produced >= count:
+                return
+            yield _FILLER_VOCABULARY[byte % len(_FILLER_VOCABULARY)]
+            produced += 1
+        block += 1
+
+
+def estimate_prefix_tokens(target_tokens: int = DEFAULT_TARGET_PREFIX_TOKENS, *, nonce: str | None = None) -> int:
+    """Estimated input tokens one probe call spends, before the model answers.
+
+    Used for the pre-run cost preview. The nonce only affects the count through
+    its fixed-width hex length, so the estimate is stable across runs.
+    """
+
+    reference_nonce = nonce if nonce is not None else "0" * (_NONCE_BYTES * 2)
+    header = _HEADER_TEMPLATE.format(nonce=reference_nonce)
+    footer = _FOOTER_TEMPLATE.format(nonce=reference_nonce)
+    lines = _filler_line_count(reference_nonce, target_tokens)
+    return _text_tokens(header) + _text_tokens(footer) + lines * _TOKENS_PER_LINE + _text_tokens(PROBE_QUESTION)
+
+
+def build_probe_prefix(nonce: str, *, target_tokens: int = DEFAULT_TARGET_PREFIX_TOKENS) -> str:
+    """The run's filler corpus. Pure in ``(nonce, target_tokens)``.
+
+    Two calls with the same nonce return identical text; two nonces share no
+    line, including the first one.
+    """
+
+    if not nonce:
+        raise ValueError("probe nonce must be a non-empty string")
+    header = _HEADER_TEMPLATE.format(nonce=nonce)
+    footer = _FOOTER_TEMPLATE.format(nonce=nonce)
+    lines = _filler_line_count(nonce, target_tokens)
+    words = _filler_words(nonce, lines * _WORDS_PER_LINE)
+    body = "\n".join(" ".join(islice(words, _WORDS_PER_LINE)) for _ in range(lines))
+    return f"{header}{body}\n{footer}"

app/modules/cache_isolation_probe/schemas.py

@@ -0,0 +1,94 @@
+"""Wire contract for the cross-account prompt cache isolation probe.
+
+Only numbers, identifiers and enumerated statuses cross this boundary. The
+generated prefix, the run nonce and every byte of upstream response content
+stay inside the service.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+from pydantic import Field
+
+from app.modules.cache_isolation_probe.service import (
+    DEFAULT_OTHER_ACCOUNTS,
+    DEFAULT_SEED_REPETITIONS,
+    MAX_OTHER_ACCOUNTS,
+    MAX_SEED_REPETITIONS,
+    ProbeCallRole,
+    ProbeCallStatus,
+    ProbeVerdict,
+)
+from app.modules.shared.schemas import DashboardModel
+
+
+class CacheProbeAccountResponse(DashboardModel):
+    account_id: str
+    label: str
+
+
+class CacheProbePressureResponse(DashboardModel):
+    under_pressure: bool
+    reason: str | None = None
+    detail: str | None = None
+    selectable_account_count: int
+    eligible_account_count: int
+    pressured_account_count: int
+
+
+class CacheProbePlanResponse(DashboardModel):
+    model: str | None = None
+    seed_account: CacheProbeAccountResponse | None = None
+    #: Every sibling the pool can offer, capped; the client picks how many of
+    #: them a run should use.
+    available_other_accounts: list[CacheProbeAccountResponse] = Field(default_factory=list)
+    seed_repetitions: int
+    total_calls: int
+    estimated_input_tokens_per_call: int
+    estimated_total_input_tokens: int
+    max_seed_repetitions: int
+    max_other_accounts: int
+    pressure: CacheProbePressureResponse
+
+
+class CacheProbeRunRequest(DashboardModel):
+    #: Explicit, typed acknowledgement that the run spends real quota. The
+    #: endpoint refuses anything but ``true`` so a bare POST cannot bill the
+    #: pool by accident.
+    confirm: bool = False
+    model: str | None = None
+    seed_repetitions: int = Field(default=DEFAULT_SEED_REPETITIONS, ge=1, le=MAX_SEED_REPETITIONS)
+    #: Upper bound on sibling accounts, not a demand: a smaller pool still
+    #: runs, with fewer rows. The plan response lists what will actually be
+    #: called.
+    other_account_count: int = Field(default=DEFAULT_OTHER_ACCOUNTS, ge=1, le=MAX_OTHER_ACCOUNTS)
+
+
+class CacheProbeCallResponse(DashboardModel):
+    sequence: int
+    account_id: str
+    account_label: str
+    role: ProbeCallRole
+    status: ProbeCallStatus
+    cache_hit: bool
+    input_tokens: int | None = None
+    cached_tokens: int | None = None
+    latency_ms: int
+    error_code: str | None = None
+
+
+class CacheProbeRunResponse(DashboardModel):
+    run_id: str
+    model: str
+    started_at: datetime
+    completed_at: datetime
+    seed_account: CacheProbeAccountResponse
+    seed_repetitions: int
+    calls: list[CacheProbeCallResponse] = Field(default_factory=list)
+    seed_hit_count: int
+    seed_call_count: int
+    other_hit_count: int
+    other_call_count: int
+    cross_account_hit: bool
+    verdict: ProbeVerdict

app/modules/cache_isolation_probe/sender.py

@@ -0,0 +1,229 @@
+"""One probe call: the routed upstream send, reported as numbers only.
+
+This deliberately reuses the same primitives the limit warm-up sender uses --
+``AuthManager.ensure_fresh`` for credentials, ``resolve_upstream_route`` for
+proxy routing, ``stream_responses`` for the upstream client -- instead of
+opening its own transport. The probe therefore inherits the account's
+credentials, its proxy binding (including fail-closed behaviour) and the
+configured stream timeouts without restating any of them.
+
+Nothing here returns or stores response content. The only values that leave
+this module are token counts, a latency, and a sanitized error code.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import time
+import uuid
+from collections.abc import AsyncGenerator
+from contextlib import AbstractAsyncContextManager
+from dataclasses import dataclass
+from typing import Callable, Protocol, cast
+
+from app.core.auth.refresh import RefreshError
+from app.core.clients.codex import CodexTransportError
+from app.core.clients.proxy import UpstreamProxyRouteTrace, stream_responses
+from app.core.crypto import TokenEncryptor
+from app.core.openai.models import ResponseUsage
+from app.core.openai.parsing import parse_sse_event
+from app.core.openai.requests import ResponsesRequest
+from app.core.upstream_proxy import ResolvedUpstreamRoute, UpstreamProxyRouteError, resolve_upstream_route
+from app.db.models import Account, AccountStatus
+from app.modules.accounts.auth_manager import AuthManager
+from app.modules.accounts.repository import AccountsRepository
+from app.modules.cache_isolation_probe.prefix import PROBE_INSTRUCTIONS, PROBE_QUESTION
+
+CACHE_PROBE_HEADER = "x-codex-lb-cache-isolation-probe"
+CACHE_PROBE_USER_AGENT = "codex-lb-cache-isolation-probe"
+#: The probe measures input caching; every output token is pure waste.
+CACHE_PROBE_MAX_OUTPUT_TOKENS = 16
+
+_TERMINAL_ERROR_EVENTS = frozenset({"response.failed", "response.incomplete", "error"})
+
+AccountsRepoFactory = Callable[[], AbstractAsyncContextManager[AccountsRepository]]
+
+
+class CacheProbeSenderPort(Protocol):
+    """What the orchestrator needs from a sender, so a test can substitute one."""
+
+    async def send(self, account_id: str, *, model: str, prefix: str) -> ProbeSendResult: ...
+
+
+@dataclass(frozen=True, slots=True)
+class ProbeSendResult:
+    """Numeric outcome of one upstream call. Never carries response content."""
+
+    ok: bool
+    latency_ms: int
+    input_tokens: int | None = None
+    cached_tokens: int | None = None
+    error_code: str | None = None
+    error_message: str | None = None
+
+
+def _elapsed_ms(started: float) -> int:
+    return int((time.monotonic() - started) * 1000)
+
+
+def _usage_tokens(usage: ResponseUsage | None) -> tuple[int | None, int | None]:
+    if usage is None:
+        return None, None
+    details = usage.input_tokens_details
+    cached = details.cached_tokens if details is not None else None
+    return usage.input_tokens, cached
+
+
+class CacheProbeSender:
+    """Sends one probe call on one named account."""
+
+    def __init__(self, accounts_repo_factory: AccountsRepoFactory) -> None:
+        self._accounts_repo_factory = accounts_repo_factory
+        self._encryptor = TokenEncryptor()
+
+    async def send(self, account_id: str, *, model: str, prefix: str) -> ProbeSendResult:
+        started = time.monotonic()
+        request_id = f"cache-probe-{uuid.uuid4().hex}"
+
+        try:
+            account = await self._refreshed_account(account_id)
+        except RefreshError as exc:
+            return ProbeSendResult(
+                ok=False,
+                latency_ms=_elapsed_ms(started),
+                error_code=f"auth_refresh_{exc.code}",
+                error_message=exc.message,
+            )
+        if account is None or account.status != AccountStatus.ACTIVE:
+            status = "missing" if account is None else account.status.value
+            return ProbeSendResult(
+                ok=False,
+                latency_ms=_elapsed_ms(started),
+                error_code="account_not_active",
+                error_message=f"Account is not active ({status})",
+            )
+
+        try:
+            route = await self._resolve_route(account)
+        except UpstreamProxyRouteError as exc:
+            return ProbeSendResult(
+                ok=False,
+                latency_ms=_elapsed_ms(started),
+                error_code="upstream_proxy_unavailable",
+                error_message=f"Upstream proxy route unavailable: {exc.reason}",
+            )
+
+        access_token = self._encryptor.decrypt(account.access_token_encrypted)
+        payload = ResponsesRequest.model_validate(
+            {
+                "model": model,
+                "instructions": PROBE_INSTRUCTIONS,
+                "input": f"{prefix}\n{PROBE_QUESTION}",
+                "tools": [],
+                "parallel_tool_calls": False,
+                "stream": True,
+                "store": False,
+                # `max_output_tokens` is stripped by `_UNSUPPORTED_UPSTREAM_FIELDS`
+                # before egress, so it cannot bound this call. It is kept because
+                # it states the intent for any model source that does honour it,
+                # and the real bound comes from the two fields below: the probe
+                # only needs `usage.input_tokens_details.cached_tokens` off the
+                # terminal frame, so every output token is waste. Without them a
+                # 28k-token prompt runs an uncapped generation, up to ten per run.
+                "max_output_tokens": CACHE_PROBE_MAX_OUTPUT_TOKENS,
+                "reasoning": {"effort": "low"},
+                "text": {"verbosity": "low"},
+            }
+        )
+        headers = {
+            "x-request-id": request_id,
+            CACHE_PROBE_HEADER: "1",
+            "user-agent": CACHE_PROBE_USER_AGENT,
+        }
+
+        usage: ResponseUsage | None = None
+        try:
+            # aclosing(), not a bare `async for`: returning mid-stream on
+            # `response.completed` would otherwise leave the upstream response
+            # and its session to async-generator finalization, overlapping the
+            # next sequential call's connection.
+            stream = stream_responses(
+                payload,
+                headers,
+                access_token,
+                account.chatgpt_account_id,
+                upstream_stream_transport_override="http",
+                route=route,
+                route_trace=UpstreamProxyRouteTrace(),
+                allow_direct_egress=route is None,
+                codex_lb_account_id=account.id,
+            )
+            # ``stream_responses`` is an async generator function; its
+            # annotation widens to ``AsyncIterator``, which does not
+            # advertise ``aclose``.
+            async with contextlib.aclosing(cast("AsyncGenerator[str, None]", stream)):
+                async for event_block in stream:
+                    event = parse_sse_event(event_block)
+                    if event is None:
+                        continue
+                    if event.response is not None and event.response.usage is not None:
+                        usage = event.response.usage
+                    if event.type == "response.completed":
+                        input_tokens, cached_tokens = _usage_tokens(usage)
+                        return ProbeSendResult(
+                            ok=True,
+                            latency_ms=_elapsed_ms(started),
+                            input_tokens=input_tokens,
+                            cached_tokens=cached_tokens,
+                        )
+                    if event.type in _TERMINAL_ERROR_EVENTS:
+                        error = event.error or (event.response.error if event.response is not None else None)
+                        input_tokens, cached_tokens = _usage_tokens(usage)
+                        return ProbeSendResult(
+                            ok=False,
+                            latency_ms=_elapsed_ms(started),
+                            input_tokens=input_tokens,
+                            cached_tokens=cached_tokens,
+                            error_code=(error.code if error is not None else None) or event.type,
+                            error_message=(error.message if error is not None else None) or event.type,
+                        )
+        except CodexTransportError as exc:
+            # Routed-transport failures carry a classified code and a message
+            # that is credential-safe by construction, so they become a typed
+            # failed row rather than an exception the orchestrator has to
+            # sanitize blindly.
+            return ProbeSendResult(
+                ok=False,
+                latency_ms=_elapsed_ms(started),
+                error_code=exc.error_code or "upstream_transport_error",
+                error_message=str(exc),
+            )
+
+        input_tokens, cached_tokens = _usage_tokens(usage)
+        return ProbeSendResult(
+            ok=False,
+            latency_ms=_elapsed_ms(started),
+            input_tokens=input_tokens,
+            cached_tokens=cached_tokens,
+            error_code="stream_incomplete",
+            error_message="Probe stream ended without a terminal event",
+        )
+
+    async def _refreshed_account(self, account_id: str) -> Account | None:
+        async with self._accounts_repo_factory() as accounts_repo:
+            current = await accounts_repo.get_by_id_fresh(account_id)
+            if current is None or current.status != AccountStatus.ACTIVE:
+                return current
+            await AuthManager(accounts_repo, refresh_repo_factory=self._accounts_repo_factory).ensure_fresh(current)
+        async with self._accounts_repo_factory() as accounts_repo:
+            return await accounts_repo.get_by_id_fresh(account_id)
+
+    async def _resolve_route(self, account: Account) -> ResolvedUpstreamRoute | None:
+        async with self._accounts_repo_factory() as accounts_repo:
+            return await resolve_upstream_route(
+                accounts_repo.session,
+                account_id=account.id,
+                operation="cache_isolation_probe",
+                scope="account",
+                encryptor=self._encryptor,
+            )