coderouter-cli 2.15.1__py3-none-any.whl → 2.17.0__py3-none-any.whl

coderouter/ingress/anthropic_errors.py

@@ -0,0 +1,204 @@
+"""Anthropic-shaped error responses for the ``/v1/messages`` ingress (v2.16.0).
+
+Before v2.16.0 a request whose whole chain failed came back as FastAPI's
+``{"detail": "profile=...: all providers failed: ..."}`` with HTTP 502,
+whatever the upstreams had said. Claude Code reads Anthropic's error shape
+(``{"type": "error", "error": {"type": ..., "message": ...}}``) and keys
+several recoveries off the upstream's *wording*:
+
+- a 400 that names a rejected beta header, ``context_management`` or tool
+  search makes it resend without that feature (2.1.288 to 2.1.305);
+- a 400 whose message says ``prompt is too long`` triggers reactive
+  compaction instead of ending the turn.
+
+None of that can fire when the message is buried in a 502 ``detail`` string.
+This module turns a :class:`NoProvidersAvailableError` into the status code,
+Anthropic error type and message the client should see, keeping the
+upstream's own words (secret-redacted).
+
+Mapping (first match wins, judged on the chain's errors):
+
+====================================  ======  ========================
+condition                             status  ``error.type``
+====================================  ======  ========================
+any error reads as context overflow   400     ``invalid_request_error``
+                                              (message starts with
+                                              ``prompt is too long:``)
+terminal error 400 / 422, not         400     ``invalid_request_error``
+retryable
+terminal error 413                    413     ``request_too_large``
+terminal error 429                    429     ``rate_limit_error``
+                                              (+ ``retry-after`` passed
+                                              through when sent)
+anything else                         502     ``api_error``
+====================================  ======  ========================
+
+v2.17.0: every mapped response also carries ``x-should-retry`` — the header
+Claude Code reads (with ``retry-after``) to decide whether to spend its retry
+budget. ``false`` for the 4xx rows (resending the same request cannot
+help), ``true`` for 429, and for 502 ``true`` only when at least one
+upstream failure was retryable — a chain that failed purely on, say, 401s
+will fail the same way on every retry.
+
+Upstream 401 / 403 deliberately fall into the last row: they describe
+CodeRouter's *own* upstream credentials, not the client's, and handing a
+401 to Claude Code would send the user to ``/login`` for nothing.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any
+
+from coderouter.secret_redaction import redact
+
+if TYPE_CHECKING:
+    from coderouter.adapters.base import AdapterError
+    from coderouter.routing.fallback import NoProvidersAvailableError
+
+
+#: Prefix Claude Code matches to start reactive compaction. Anthropic's own
+#: wording is ``prompt is too long: 208000 tokens > 200000 maximum``.
+PROMPT_TOO_LONG_PREFIX = "prompt is too long"
+
+# Upstream wordings for "the prompt does not fit the context window".
+# Sources (2026-10): llama.cpp server ``exceed_context_size_error`` /
+# "the request exceeds the available context size"; vLLM / OpenAI
+# "This model's maximum context length is N tokens"; Ollama "prompt too
+# long"; OpenAI ``context_length_exceeded``; Anthropic "prompt is too long".
+_CONTEXT_OVERFLOW_RE = re.compile(
+    r"exceed_context_size"
+    r"|exceeds? the available context"
+    r"|context_length_exceeded"
+    r"|maximum context length"
+    r"|prompt is too long"
+    r"|prompt too long"
+    r"|context window (?:is )?(?:exceeded|full)"
+    r"|exceeds? (?:the )?(?:model'?s )?context (?:length|window|size)",
+    re.IGNORECASE,
+)
+
+_MAX_MESSAGE_CHARS = 2000
+
+
+@dataclass(frozen=True)
+class AnthropicErrorResponse:
+    """What the ingress should send for a failed chain."""
+
+    status_code: int
+    error_type: str
+    message: str
+    headers: dict[str, str] = field(default_factory=dict)
+
+    def body(self) -> dict[str, Any]:
+        """Anthropic error envelope."""
+        return {
+            "type": "error",
+            "error": {"type": self.error_type, "message": self.message},
+        }
+
+
+def upstream_message(err: AdapterError) -> str:
+    """Best human-readable message for one upstream failure.
+
+    Pulls ``error.message`` out of Anthropic / OpenAI / llama.cpp JSON
+    bodies, falls back to the raw body text, then to the adapter's own
+    message. Always secret-redacted and length-capped.
+    """
+    body = getattr(err, "upstream_body", None)
+    text: str | None = None
+    if body:
+        text = _message_from_body(body)
+    if not text:
+        text = Exception.__str__(err)
+    text = redact(text.strip())
+    if len(text) > _MAX_MESSAGE_CHARS:
+        text = text[:_MAX_MESSAGE_CHARS] + "…"
+    return text
+
+
+def _message_from_body(body: str) -> str | None:
+    try:
+        parsed = json.loads(body)
+    except (json.JSONDecodeError, ValueError):
+        stripped = body.strip()
+        return stripped or None
+    if isinstance(parsed, dict):
+        err = parsed.get("error")
+        if isinstance(err, dict):
+            msg = err.get("message")
+            if isinstance(msg, str) and msg:
+                return msg
+        if isinstance(err, str) and err:
+            return err
+        for key in ("message", "detail"):
+            val = parsed.get(key)
+            if isinstance(val, str) and val:
+                return val
+    return body.strip() or None
+
+
+# "prompt + max_tokens > window": the *completion* budget is what is too big.
+# vLLM / OpenAI: "... However, you requested 33000 tokens (1000 in the
+# messages, 32000 in the completion)". Compacting the prompt cannot fix that,
+# so it must not be reported as "prompt is too long".
+_COMPLETION_BUDGET_RE = re.compile(
+    r"in the completion|max_tokens|max_completion_tokens|n_predict",
+    re.IGNORECASE,
+)
+
+
+def is_context_overflow(err: AdapterError) -> bool:
+    """True when the upstream said the *prompt* does not fit its context."""
+    if err.status_code is not None and err.status_code not in (400, 413, 422):
+        return False
+    haystack = " ".join(
+        part for part in (getattr(err, "upstream_body", None), Exception.__str__(err)) if part
+    )
+    if not _CONTEXT_OVERFLOW_RE.search(haystack):
+        return False
+    return not _COMPLETION_BUDGET_RE.search(haystack)
+
+
+def map_no_providers_error(exc: NoProvidersAvailableError) -> AnthropicErrorResponse:
+    """Decide status / type / message for a chain where every provider failed."""
+    errors = list(exc.errors)
+
+    for err in errors:
+        if is_context_overflow(err):
+            msg = upstream_message(err)
+            if not msg.lower().startswith(PROMPT_TOO_LONG_PREFIX):
+                msg = f"{PROMPT_TOO_LONG_PREFIX}: {msg}"
+            return AnthropicErrorResponse(
+                400, "invalid_request_error", msg, {"x-should-retry": "false"}
+            )
+
+    terminal = errors[-1] if errors else None
+    if terminal is not None and terminal.status_code is not None:
+        status = terminal.status_code
+        if status in (400, 422) and not terminal.retryable:
+            return AnthropicErrorResponse(
+                400,
+                "invalid_request_error",
+                upstream_message(terminal),
+                {"x-should-retry": "false"},
+            )
+        if status == 413:
+            return AnthropicErrorResponse(
+                413, "request_too_large", upstream_message(terminal), {"x-should-retry": "false"}
+            )
+        if status == 429:
+            headers: dict[str, str] = {"x-should-retry": "true"}
+            retry_after = getattr(terminal, "retry_after", None)
+            if retry_after:
+                headers["retry-after"] = retry_after
+            return AnthropicErrorResponse(
+                429, "rate_limit_error", upstream_message(terminal), headers
+            )
+
+    retry = any(e.retryable for e in errors)
+    return AnthropicErrorResponse(
+        502, "api_error", redact(str(exc)), {"x-should-retry": "true" if retry else "false"}
+    )

coderouter/ingress/anthropic_routes.py

@@ -28,10 +28,21 @@ from collections.abc import AsyncIterator
 from typing import Any
 
 from fastapi import APIRouter, Header, HTTPException, Request
-from fastapi.responses import JSONResponse, StreamingResponse
+from fastapi.responses import JSONResponse, Response, StreamingResponse
 
 from coderouter.adapters.base import StreamTruncatedError
+from coderouter.guards.context_budget import ContextBudgetRejectError
 from coderouter.guards.tool_loop import ToolCountExceededError, ToolLoopBreakError
+from coderouter.ingress.anthropic_errors import (
+    AnthropicErrorResponse,
+    map_no_providers_error,
+)
+from coderouter.ingress.tool_search import (
+    REJECT_MESSAGE,
+    expand_tool_search,
+    resolve_action,
+    uses_tool_search,
+)
 from coderouter.logging import get_logger
 from coderouter.routing import (
     FallbackEngine,
@@ -115,6 +126,7 @@ async def _guard_stream(
     *,
     timeout_s: float,
     label: str,
+    deadline: float | None = None,
 ) -> AsyncIterator[str]:
     """M14: enforce an overall timeout + clean cancellation on an SSE stream.
 
@@ -135,7 +147,10 @@ async def _guard_stream(
     wrapper only has to deal with timeout / cancellation / normal completion.
     """
     try:
-        async with asyncio.timeout(timeout_s):
+        # v2.16.0: ``deadline`` (loop time) lets the caller share one overall
+        # ceiling with work it did before the stream started.
+        when = deadline if deadline is not None else asyncio.get_running_loop().time() + timeout_s
+        async with asyncio.timeout_at(when):
             async for frame in source:
                 yield frame
     except TimeoutError:
@@ -177,7 +192,7 @@ async def messages(
     x_coderouter_mode: str | None = Header(default=None, alias=_MODE_HEADER),
     anthropic_version: str | None = Header(default=None, alias=_ANTHROPIC_VERSION_HEADER),
     anthropic_beta: str | None = Header(default=None, alias=_ANTHROPIC_BETA_HEADER),
-) -> StreamingResponse | dict[str, Any]:
+) -> Response | dict[str, Any]:
     """Anthropic Messages API endpoint.
 
     Validates the body into :class:`AnthropicRequest`, resolves the
@@ -250,10 +265,37 @@ async def messages(
                 detail=(f"unknown profile {anth_req.profile!r}. available: {available}"),
             ) from exc
 
+    # v2.16.0: Claude Code tool search (``defer_loading`` tools plus a
+    # server-side ``tool_search_tool_*``) only works on api.anthropic.com.
+    # See ``coderouter.ingress.tool_search``.
+    if uses_tool_search(anth_req):
+        ts_action = resolve_action(config, anth_req.profile)
+        logger.info(
+            "tool-search-request",
+            extra={"action": ts_action, "profile": anth_req.profile or config.default_profile},
+        )
+        if ts_action == "reject":
+            return _anthropic_error_response(
+                AnthropicErrorResponse(
+                    400, "invalid_request_error", REJECT_MESSAGE, {"x-should-retry": "false"}
+                )
+            )
+        if ts_action == "expand":
+            anth_req = expand_tool_search(anth_req)
+
     # v2.0-F (L1): run context budget guard before dispatch so the
     # response header can be set for both streaming and non-streaming.
     # The engine's internal guard re-check is a cheap no-op.
-    anth_req, ctx_budget_status = engine.apply_context_budget(anth_req)
+    try:
+        anth_req, ctx_budget_status = engine.apply_context_budget(anth_req)
+    except ContextBudgetRejectError as exc:
+        # v2.16.0: ``context_budget_action: reject`` — Anthropic's own
+        # overflow answer, so Claude Code compacts the conversation.
+        return _anthropic_error_response(
+            AnthropicErrorResponse(
+                400, "invalid_request_error", str(exc), {"x-should-retry": "false"}
+            )
+        )
 
     if anth_req.stream:
         stream_headers: dict[str, str] = {
@@ -286,9 +328,86 @@ async def messages(
         # disconnect cleanup so a wedged upstream cannot pin the client
         # (or the upstream socket) open forever.
         timeout_s = _resolve_stream_timeout_s(engine, anth_req.profile)
+
+        # v2.16.0: pull the engine's first event *before* committing the
+        # HTTP response. A chain that fails before producing anything (every
+        # provider refused, an upstream said the prompt is too long, a 429)
+        # then gets a real HTTP status and an Anthropic error body — the
+        # same thing api.anthropic.com does — instead of ``200`` plus an
+        # in-stream ``error`` event. Clients key retries and reactive
+        # compaction off that status. Once the first event exists, nothing
+        # changes: the stream is committed exactly as before.
+        source = engine.stream_anthropic(anth_req)
+        first_event: AnthropicStreamEvent | None = None
+        pending_exc: BaseException | None = None
+        # One deadline for the whole stream, prefetch included, so the M14
+        # ceiling stays an *overall* ceiling (review fix: two independent
+        # ``timeout_s`` windows allowed ~2x).
+        deadline = asyncio.get_running_loop().time() + timeout_s
+        # Starlette cancels a StreamingResponse body when the client leaves,
+        # but not an endpoint that is still awaiting. The prefetch runs in
+        # the endpoint, so watch for the disconnect ourselves and cancel the
+        # wait — otherwise a user pressing Esc during a long local prefill
+        # would leave the upstream (and its llama.cpp slot) busy until the
+        # first token arrived.
+        watcher = _DisconnectWatcher(request)
+        watcher.start()
+        try:
+            async with asyncio.timeout_at(deadline) as prefetch_scope:
+                first_event = await anext(source)
+        except StopAsyncIteration:
+            first_event = None
+        except NoProvidersAvailableError as exc:
+            await _aclose_quietly(source)
+            return _anthropic_error_response(map_no_providers_error(exc))
+        except ContextBudgetRejectError as exc:
+            # The engine re-runs the budget guard when the prepared dispatch
+            # is not reused (input-filter plugins, tool-loop inject, model
+            # swap) — same answer as the pre-dispatch check above.
+            await _aclose_quietly(source)
+            return _anthropic_error_response(
+                AnthropicErrorResponse(
+                400, "invalid_request_error", str(exc), {"x-should-retry": "false"}
+            )
+            )
+        except (ToolLoopBreakError, ToolCountExceededError, MidStreamError) as exc:
+            # Policy errors and mid-stream failures keep their existing
+            # in-stream framing; the iterator below re-raises them.
+            pending_exc = exc
+        except TimeoutError:
+            await _aclose_quietly(source)
+            if not prefetch_scope.expired():
+                # A TimeoutError raised *inside* the engine is not our
+                # ceiling — do not dress it up as one.
+                raise
+            return _anthropic_error_response(
+                AnthropicErrorResponse(
+                    504, "timeout_error", f"no upstream event within {timeout_s:.0f}s"
+                )
+            )
+        except asyncio.CancelledError:
+            await _aclose_quietly(source)
+            if watcher.fired:
+                logger.info("sse-client-disconnect", extra={"label": "anthropic-prefetch"})
+                watcher.uncancel()
+                return Response(status_code=499)
+            raise
+        except BaseException:
+            await _aclose_quietly(source)
+            raise
+        finally:
+            await watcher.stop()
+
         guarded = _guard_stream(
-            _anthropic_sse_iterator(engine, anth_req),
+            _anthropic_sse_iterator(
+                engine,
+                anth_req,
+                source=source,
+                first_event=first_event,
+                pending_exc=pending_exc,
+            ),
             timeout_s=timeout_s,
+            deadline=deadline,
             label="anthropic",
         )
         return StreamingResponse(
@@ -299,8 +418,18 @@ async def messages(
 
     try:
         anth_resp = await engine.generate_anthropic(anth_req)
+    except ContextBudgetRejectError as exc:
+        # See the streaming branch: the engine can re-run the guard.
+        return _anthropic_error_response(
+            AnthropicErrorResponse(
+                400, "invalid_request_error", str(exc), {"x-should-retry": "false"}
+            )
+        )
     except NoProvidersAvailableError as exc:
-        raise HTTPException(status_code=502, detail=str(exc)) from exc
+        # v2.16.0: Anthropic-shaped error with the upstream's own wording
+        # and a status that reflects it (400 / 413 / 429 / 502). See
+        # ``coderouter.ingress.anthropic_errors``.
+        return _anthropic_error_response(map_no_providers_error(exc))
     except ToolLoopBreakError as exc:
         # v1.9-E (L3): the ``break`` action short-circuits the request
         # before any provider is called. Surface as a structured 400 so
@@ -457,8 +586,81 @@ async def count_tokens_route(
     return {"input_tokens": input_tokens}
 
 
+class _DisconnectWatcher:
+    """Cancel the current task when the HTTP client disconnects.
+
+    Used only around the streaming prefetch (see ``messages``). Polls
+    ``request.is_disconnected()`` — a non-blocking check — every
+    ``interval`` seconds from a side task and, on disconnect, cancels the
+    task that created the watcher. ``fired`` tells the caller that a
+    ``CancelledError`` came from here (and ``uncancel`` clears it), as
+    opposed to a real cancellation from the server.
+    """
+
+    def __init__(self, request: Request, interval: float = 0.25) -> None:
+        self._request = request
+        self._interval = interval
+        self._owner = asyncio.current_task()
+        self._task: asyncio.Task[None] | None = None
+        self.fired = False
+
+    def start(self) -> None:
+        if self._owner is not None:
+            self._task = asyncio.create_task(self._run())
+
+    async def _run(self) -> None:
+        while True:
+            await asyncio.sleep(self._interval)
+            try:
+                gone = await self._request.is_disconnected()
+            except Exception:
+                return
+            if gone:
+                self.fired = True
+                if self._owner is not None:
+                    self._owner.cancel()
+                return
+
+    def uncancel(self) -> None:
+        if self._owner is not None:
+            self._owner.uncancel()
+
+    async def stop(self) -> None:
+        if self._task is not None and not self._task.done():
+            self._task.cancel()
+            with contextlib.suppress(asyncio.CancelledError, Exception):
+                await self._task
+
+
+def _anthropic_error_response(mapped: AnthropicErrorResponse) -> JSONResponse:
+    """Render a mapped failure, carrying the v2.15.0 fallback headers."""
+    headers = dict(mapped.headers)
+    trace = current_fallback_trace()
+    if trace is not None:
+        headers.update(trace.header_values())
+    logger.info(
+        "anthropic-error-response",
+        extra={"status": mapped.status_code, "error_type": mapped.error_type},
+    )
+    return JSONResponse(
+        content=mapped.body(), status_code=mapped.status_code, headers=headers or None
+    )
+
+
+async def _aclose_quietly(source: AsyncIterator[Any]) -> None:
+    aclose = getattr(source, "aclose", None)
+    if aclose is not None:
+        with contextlib.suppress(Exception):
+            await aclose()
+
+
 async def _anthropic_sse_iterator(
-    engine: FallbackEngine, anth_req: AnthropicRequest
+    engine: FallbackEngine,
+    anth_req: AnthropicRequest,
+    *,
+    source: AsyncIterator[AnthropicStreamEvent] | None = None,
+    first_event: AnthropicStreamEvent | None = None,
+    pending_exc: BaseException | None = None,
 ) -> AsyncIterator[str]:
     """Serialize engine.stream_anthropic() onto the Anthropic SSE wire.
 
@@ -466,9 +668,20 @@ async def _anthropic_sse_iterator(
     Anthropic spec (distinct from OpenAI's `data:`-only format).
     Errors map to in-stream `event: error` events — we never switch an
     in-flight HTTP response to a 5xx once headers have shipped.
+
+    v2.16.0: the route may have already pulled the first event (to answer
+    pre-content failures with a real HTTP status). It then passes the
+    started ``source``, that ``first_event``, and any exception the pull
+    raised (``pending_exc``), which is re-raised here so its existing
+    in-stream framing below applies unchanged.
     """
+    events = source if source is not None else engine.stream_anthropic(anth_req)
     try:
-        async for ev in engine.stream_anthropic(anth_req):
+        if pending_exc is not None:
+            raise pending_exc
+        if first_event is not None:
+            yield _format_anthropic_sse(first_event)
+        async for ev in events:
             yield _format_anthropic_sse(ev)
     except NoProvidersAvailableError as exc:
         # No provider produced even the first event — surface as overloaded.

coderouter/ingress/tool_search.py

@@ -0,0 +1,100 @@
+"""Claude Code tool search on non-Anthropic upstreams (v2.16.0).
+
+Claude Code 2.1.304 turned tool search on by default behind a custom
+``ANTHROPIC_BASE_URL``. A tool-search request carries:
+
+- a server-side search tool, ``{"type": "tool_search_tool_regex_...",
+  "name": "tool_search_tool_regex"}`` (or the ``bm25`` variant), which only
+  api.anthropic.com executes, and
+- ordinary tools marked ``"defer_loading": true`` that the API keeps out
+  of the model's context until a search surfaces them.
+
+A local model behind CodeRouter cannot run the search: before v2.16.0 the
+search tool reached it as an ordinary function it could "call", and every
+deferred tool was loaded anyway. Claude Code 2.1.305 retries a request
+without tool search when the gateway answers 400 / 422, so rejecting is the
+cheapest correct answer. Which action applies is the profile's
+``deferred_tools_action`` (see ``FallbackChain``).
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Literal
+from urllib.parse import urlparse
+
+if TYPE_CHECKING:
+    from coderouter.config.schemas import CodeRouterConfig
+    from coderouter.translation import AnthropicRequest
+
+TOOL_SEARCH_TYPE_PREFIX = "tool_search_tool"
+
+#: Message for the 400. Mentions both trigger words so a client (or a human
+#: reading the log) can tell why tool search was refused.
+REJECT_MESSAGE = (
+    "tool search is not supported by this gateway's upstreams: remove the "
+    "tool_search_tool and defer_loading (CodeRouter deferred_tools_action)"
+)
+
+Action = Literal["reject", "expand", "passthrough"]
+
+
+def _tool_dict(tool: Any) -> dict[str, Any]:
+    if hasattr(tool, "model_dump"):
+        dumped: dict[str, Any] = tool.model_dump()
+        return dumped
+    return dict(tool)
+
+
+def _is_search_tool(t: dict[str, Any]) -> bool:
+    kind = t.get("type")
+    return isinstance(kind, str) and kind.startswith(TOOL_SEARCH_TYPE_PREFIX)
+
+
+def uses_tool_search(req: AnthropicRequest) -> bool:
+    """True when the request carries a search tool or a deferred tool."""
+    for tool in req.tools or []:
+        t = _tool_dict(tool)
+        if _is_search_tool(t) or t.get("defer_loading") is True:
+            return True
+    return False
+
+
+def _is_anthropic_api(provider: Any) -> bool:
+    if getattr(provider, "kind", None) != "anthropic":
+        return False
+    host = urlparse(str(getattr(provider, "base_url", ""))).hostname or ""
+    return host == "anthropic.com" or host.endswith(".anthropic.com")
+
+
+def resolve_action(config: CodeRouterConfig, profile_name: str | None) -> Action:
+    """Resolve the profile's ``deferred_tools_action`` (``auto`` → concrete)."""
+    try:
+        profile = config.profile_by_name(profile_name or config.default_profile)
+    except (KeyError, ValueError):
+        return "reject"
+    action = getattr(profile, "deferred_tools_action", "auto")
+    if action == "reject":
+        return "reject"
+    if action == "expand":
+        return "expand"
+    if action == "passthrough":
+        return "passthrough"
+    providers = {p.name: p for p in config.providers}
+    chain = [providers.get(name) for name in profile.providers]
+    if chain and all(p is not None and _is_anthropic_api(p) for p in chain):
+        return "passthrough"
+    return "reject"
+
+
+def expand_tool_search(req: AnthropicRequest) -> AnthropicRequest:
+    """Drop the search tool(s) and every ``defer_loading`` flag."""
+    from coderouter.translation.anthropic import AnthropicTool
+
+    kept: list[AnthropicTool] = []
+    for tool in req.tools or []:
+        t = _tool_dict(tool)
+        if _is_search_tool(t):
+            continue
+        t.pop("defer_loading", None)
+        kept.append(AnthropicTool.model_validate(t))
+    return req.model_copy(update={"tools": kept or None})