coderouter-cli 2.16.0__tar.gz → 2.17.0__tar.gz

{coderouter_cli-2.16.0 → coderouter_cli-2.17.0}/CHANGELOG.md

@@ -9,6 +9,53 @@ are kept verbatim where the Japanese text itself is the subject).
 
 ---
 
+## [v2.17.0] — 2026-10-24 (time-to-first-token cut-off, retry signals)
+
+### Added
+
+- **`ttft_timeout_s` (provider).** A streaming attempt that has not
+  produced its first real output (text / tool_use / thinking delta) within
+  this many seconds is abandoned before any byte reached the client, and
+  the chain moves on — e.g. a local llama-server still loading a model or
+  prefilling a long prompt while a cloud provider could answer now. Never
+  applied to the last provider of a chain (nothing to fall back to), nor to
+  non-streaming requests (`timeout_s` covers those), nor to the
+  `/v1/chat/completions` route. It is also skipped for the tool-turn
+  downgrade (`kind: openai_compat` with tools, which calls the upstream
+  non-streaming so tool calls can be repaired): there the "first output" is
+  the whole answer, and a deadline would discard slow but healthy replies —
+  use `kind: anthropic` (native `/v1/messages`) to get the cut-off for a
+  local server. It always covers the
+  wait for the first event; covering the gap between the opening events and
+  the first real output needs `empty_response_action: fallback`, which is
+  what withholds those events from the client. Abandoned attempts are
+  ordinary retryable failures: new fallback reason `ttft-timeout` (detail
+  `no-first-token`) and the self-healing guards learn from them. A cut that
+  happens after the opening events is handled like a truncated stream: the
+  withheld, unterminated preamble is never flushed to the client if the
+  rest of the chain also fails, and the abandoned upstream stream is closed
+  immediately. Default
+  `None` (off). Docs: `docs/concepts/commit-point.md` / `.en.md`.
+- **`x-should-retry` on `/v1/messages` errors.** Every Anthropic-shaped
+  error CodeRouter produces now says whether retrying can help — the header
+  Claude Code reads, with `retry-after`, to decide whether to spend its
+  retry budget: `false` for 400 / 413 (including tool-search and
+  `context_budget_action: reject` refusals), `true` for 429, and for 502
+  `true` only when at least one upstream failure was retryable (a chain that
+  failed purely on 401/403 fails the same way on every retry).
+- **Docs: the commit point** (`docs/concepts/commit-point.md` / `.en.md`) —
+  which failures before the body ships CodeRouter can still move to another
+  provider, which it cannot, and the error each chain failure becomes.
+  Recommends `empty_response_action: fallback` for Claude Code profiles.
+
+### Known limits
+
+- No keep-alive `ping` is sent while opening events are withheld; keep
+  `ttft_timeout_s` below the client's idle timeout
+  (`CLAUDE_STREAM_IDLE_TIMEOUT_MS` for Claude Code).
+
+---
+
 ## [v2.16.0] — 2026-10-24 (Claude Code 2.1.3xx behind a gateway)
 
 Claude Code 2.1.285–2.1.309 changed several defaults for sessions behind a

{coderouter_cli-2.16.0 → coderouter_cli-2.17.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: coderouter-cli
-Version: 2.16.0
+Version: 2.17.0
 Summary: Local-first, free-first, fallback-built-in LLM router. Claude Code / OpenAI compatible.
 Project-URL: Homepage, https://github.com/zephel01/CodeRouter
 Project-URL: Repository, https://github.com/zephel01/CodeRouter

{coderouter_cli-2.16.0 → coderouter_cli-2.17.0}/coderouter/adapters/base.py

@@ -200,6 +200,23 @@ class StreamTruncatedError(AdapterError):
         self.tool_call_in_flight = tool_call_in_flight
 
 
+# v2.17.0: the provider did not produce its first real streamed output within
+# ``ProviderConfig.ttft_timeout_s``. Raised by the engine (not the adapter),
+# before any byte reached the client, so the chain falls back normally and the
+# self-healing guards learn from it like any other retryable failure.
+class TTFTTimeoutError(AdapterError):
+    """No first token within ``ttft_timeout_s``."""
+
+    def __init__(self, *, provider: str, ttft_timeout_s: float) -> None:
+        super().__init__(
+            f"timeout: no first token within {ttft_timeout_s:g}s (ttft_timeout_s)",
+            provider=provider,
+            status_code=None,
+            retryable=True,
+        )
+        self.ttft_timeout_s = ttft_timeout_s
+
+
 # v0.6-B: per-call overrides resolved from the active profile. The engine
 # builds one instance per request (since a profile is invariant across its
 # chain) and threads it through every adapter call on that chain. Adapters

{coderouter_cli-2.16.0 → coderouter_cli-2.17.0}/coderouter/config/schemas.py

@@ -550,6 +550,28 @@ class ProviderConfig(BaseModel):
         description="If true, only used when ALLOW_PAID=true (plan.md §2.3).",
     )
     timeout_s: float = Field(default=30.0, ge=1.0, le=86400.0)
+    # v2.17.0: time-to-first-token cut-off for streaming requests. When the
+    # provider has not produced its first real output (text / tool_use /
+    # thinking delta) within this many seconds of the attempt starting, the
+    # attempt is abandoned and the chain moves on — before any byte reached
+    # the client. Never applied to the last provider of a chain (nothing to
+    # move on to), nor to non-streaming requests (``timeout_s`` covers
+    # those). Before the first *event* it always applies; between the first
+    # event and the first real output it needs
+    # ``empty_response_action: fallback`` on the profile, which is what
+    # withholds those opening events from the client. ``None`` (default)
+    # disables it. Typical use: a local llama-server still loading a model
+    # or chewing through a long prompt while a cloud fallback could answer.
+    ttft_timeout_s: float | None = Field(
+        default=None,
+        gt=0.0,
+        le=3600.0,
+        description=(
+            "v2.17.0: seconds to wait for the first real streamed output "
+            "before falling back to the next provider (not applied to the "
+            "last provider in a chain). None disables."
+        ),
+    )
 
     # v2.6 language-tax track: path to a LOCAL ``tokenizer.json`` for this
     # provider's model, used to measure the CJK over-count vs the char/4

{coderouter_cli-2.16.0 → coderouter_cli-2.17.0}/coderouter/ingress/anthropic_errors.py

@@ -33,6 +33,13 @@ terminal error 429                    429     ``rate_limit_error``
 anything else                         502     ``api_error``
 ====================================  ======  ========================
 
+v2.17.0: every mapped response also carries ``x-should-retry`` — the header
+Claude Code reads (with ``retry-after``) to decide whether to spend its retry
+budget. ``false`` for the 4xx rows (resending the same request cannot
+help), ``true`` for 429, and for 502 ``true`` only when at least one
+upstream failure was retryable — a chain that failed purely on, say, 401s
+will fail the same way on every retry.
+
 Upstream 401 / 403 deliberately fall into the last row: they describe
 CodeRouter's *own* upstream credentials, not the client's, and handing a
 401 to Claude Code would send the user to ``/login`` for nothing.
@@ -164,17 +171,26 @@ def map_no_providers_error(exc: NoProvidersAvailableError) -> AnthropicErrorResp
             msg = upstream_message(err)
             if not msg.lower().startswith(PROMPT_TOO_LONG_PREFIX):
                 msg = f"{PROMPT_TOO_LONG_PREFIX}: {msg}"
-            return AnthropicErrorResponse(400, "invalid_request_error", msg)
+            return AnthropicErrorResponse(
+                400, "invalid_request_error", msg, {"x-should-retry": "false"}
+            )
 
     terminal = errors[-1] if errors else None
     if terminal is not None and terminal.status_code is not None:
         status = terminal.status_code
         if status in (400, 422) and not terminal.retryable:
-            return AnthropicErrorResponse(400, "invalid_request_error", upstream_message(terminal))
+            return AnthropicErrorResponse(
+                400,
+                "invalid_request_error",
+                upstream_message(terminal),
+                {"x-should-retry": "false"},
+            )
         if status == 413:
-            return AnthropicErrorResponse(413, "request_too_large", upstream_message(terminal))
+            return AnthropicErrorResponse(
+                413, "request_too_large", upstream_message(terminal), {"x-should-retry": "false"}
+            )
         if status == 429:
-            headers: dict[str, str] = {}
+            headers: dict[str, str] = {"x-should-retry": "true"}
             retry_after = getattr(terminal, "retry_after", None)
             if retry_after:
                 headers["retry-after"] = retry_after
@@ -182,4 +198,7 @@ def map_no_providers_error(exc: NoProvidersAvailableError) -> AnthropicErrorResp
                 429, "rate_limit_error", upstream_message(terminal), headers
             )
 
-    return AnthropicErrorResponse(502, "api_error", redact(str(exc)))
+    retry = any(e.retryable for e in errors)
+    return AnthropicErrorResponse(
+        502, "api_error", redact(str(exc)), {"x-should-retry": "true" if retry else "false"}
+    )

{coderouter_cli-2.16.0 → coderouter_cli-2.17.0}/coderouter/ingress/anthropic_routes.py

@@ -276,7 +276,9 @@ async def messages(
         )
         if ts_action == "reject":
             return _anthropic_error_response(
-                AnthropicErrorResponse(400, "invalid_request_error", REJECT_MESSAGE)
+                AnthropicErrorResponse(
+                    400, "invalid_request_error", REJECT_MESSAGE, {"x-should-retry": "false"}
+                )
             )
         if ts_action == "expand":
             anth_req = expand_tool_search(anth_req)
@@ -290,7 +292,9 @@ async def messages(
         # v2.16.0: ``context_budget_action: reject`` — Anthropic's own
         # overflow answer, so Claude Code compacts the conversation.
         return _anthropic_error_response(
-            AnthropicErrorResponse(400, "invalid_request_error", str(exc))
+            AnthropicErrorResponse(
+                400, "invalid_request_error", str(exc), {"x-should-retry": "false"}
+            )
         )
 
     if anth_req.stream:
@@ -362,7 +366,9 @@ async def messages(
             # swap) — same answer as the pre-dispatch check above.
             await _aclose_quietly(source)
             return _anthropic_error_response(
-                AnthropicErrorResponse(400, "invalid_request_error", str(exc))
+                AnthropicErrorResponse(
+                400, "invalid_request_error", str(exc), {"x-should-retry": "false"}
+            )
             )
         except (ToolLoopBreakError, ToolCountExceededError, MidStreamError) as exc:
             # Policy errors and mid-stream failures keep their existing
@@ -415,7 +421,9 @@ async def messages(
     except ContextBudgetRejectError as exc:
         # See the streaming branch: the engine can re-run the guard.
         return _anthropic_error_response(
-            AnthropicErrorResponse(400, "invalid_request_error", str(exc))
+            AnthropicErrorResponse(
+                400, "invalid_request_error", str(exc), {"x-should-retry": "false"}
+            )
         )
     except NoProvidersAvailableError as exc:
         # v2.16.0: Anthropic-shaped error with the upstream's own wording

{coderouter_cli-2.16.0 → coderouter_cli-2.17.0}/coderouter/routing/fallback.py

@@ -47,6 +47,7 @@ from coderouter.adapters.base import (
     ProviderCallOverrides,
     StreamChunk,
     StreamTruncatedError,
+    TTFTTimeoutError,
 )
 from coderouter.adapters.registry import build_adapter
 from coderouter.config.schemas import CodeRouterConfig, ProviderConfig
@@ -111,6 +112,7 @@ from coderouter.routing.fallback_trace import (
     REASON_PAID_GATE,
     REASON_SELF_HEALING_EXCLUDED,
     REASON_STREAM_TRUNCATED,
+    REASON_TTFT_TIMEOUT,
     REASON_UNKNOWN_PROVIDER,
     FallbackTrace,
     begin_fallback_trace,
@@ -898,6 +900,38 @@ def _emit_cache_observed_streaming(
     )
 
 
+async def _await_within_ttft(
+    awaitable: Any,
+    *,
+    deadline: float | None,
+    provider: str,
+    ttft_timeout_s: float | None,
+) -> Any:
+    """v2.17.0: await ``awaitable`` but give up at the TTFT ``deadline``.
+
+    ``deadline`` is a ``time.monotonic()`` instant (``None`` = no limit). On
+    expiry raises :class:`TTFTTimeoutError` — a retryable ``AdapterError``, so
+    every existing pre-content fallback branch handles it unchanged. The
+    cancelled await unwinds the adapter's generator, which releases its
+    upstream connection.
+    """
+    if deadline is None:
+        return await awaitable
+    remaining = deadline - time.monotonic()
+    if remaining <= 0:
+        close = getattr(awaitable, "close", None)
+        if close is not None:
+            close()
+        raise TTFTTimeoutError(provider=provider, ttft_timeout_s=ttft_timeout_s or 0.0)
+    try:
+        async with asyncio.timeout(remaining):
+            return await awaitable
+    except TimeoutError as exc:
+        raise TTFTTimeoutError(
+            provider=provider, ttft_timeout_s=ttft_timeout_s or 0.0
+        ) from exc
+
+
 class NoProvidersAvailableError(CodeRouterError):
     """Raised when every provider in the chain has failed (or was filtered out)."""
 
@@ -3354,7 +3388,7 @@ class FallbackEngine:
         # keeps the default byte-for-byte identical to v2.14.0.
         last_empty_stream_truncated = False
 
-        for adapter, will_degrade in chain:
+        for chain_idx, (adapter, will_degrade) in enumerate(chain):
             is_native = isinstance(adapter, AnthropicAdapter)
             downgrading = (not is_native) and bool(request.tools)
             effective_request = request
@@ -3414,6 +3448,14 @@ class FallbackEngine:
             # only the non-streaming generate_anthropic recorded, so the
             # busiest path produced no adaptive signal.
             attempt_started = time.monotonic()
+            # v2.17.0: TTFT cut-off. Never on the last provider — there is
+            # nothing left to fall back to, so waiting is the better answer.
+            ttft_s = getattr(adapter.config, "ttft_timeout_s", None)
+            ttft_deadline = (
+                attempt_started + ttft_s
+                if ttft_s is not None and chain_idx < len(chain) - 1
+                else None
+            )
             event_iter: AsyncIterator[AnthropicStreamEvent]
             first: AnthropicStreamEvent
             try:
@@ -3422,11 +3464,21 @@ class FallbackEngine:
                 # mypy narrows for stream_anthropic (not on BaseAdapter).
                 if isinstance(adapter, AnthropicAdapter):
                     event_iter = adapter.stream_anthropic(effective_request, overrides=overrides)
-                    first = await anext(event_iter)
+                    first = await _await_within_ttft(
+                        anext(event_iter),
+                        deadline=ttft_deadline,
+                        provider=adapter.name,
+                        ttft_timeout_s=ttft_s,
+                    )
                 elif downgrading:
                     # v0.3-D downgrade: run non-streaming, repair, replay.
                     chat_req = to_chat_request(effective_request)
                     chat_req.stream = False
+                    # v2.17.0: no TTFT deadline here. This branch is a
+                    # *non-streaming* call (the tool-turn downgrade), so its
+                    # "first output" is the whole answer — a TTFT cut would
+                    # become a total-generation limit and throw away slow
+                    # but healthy local replies. ``timeout_s`` bounds it.
                     chat_resp = await adapter.generate(chat_req, overrides=overrides)
                     # H-4: same as the non-streaming branch — keep the
                     # CPU-bound repair scan off the event loop.
@@ -3441,7 +3493,12 @@ class FallbackEngine:
                     event_iter = stream_chat_to_anthropic_events(
                         adapter.stream(chat_req, overrides=overrides)
                     )
-                    first = await anext(event_iter)
+                    first = await _await_within_ttft(
+                        anext(event_iter),
+                        deadline=ttft_deadline,
+                        provider=adapter.name,
+                        ttft_timeout_s=ttft_s,
+                    )
             except StopAsyncIteration:
                 # M2: record the empty-stream failure.
                 self._adaptive.record_attempt(
@@ -3557,7 +3614,18 @@ class FallbackEngine:
                                 yield buffered_ev
                             buffer = []
                     try:
-                        pending = await iterator.__anext__()
+                        if content_started:
+                            pending = await iterator.__anext__()
+                        else:
+                            # v2.17.0: the TTFT deadline also covers the
+                            # withheld preamble — first *real* output is
+                            # what counts, not the first event.
+                            pending = await _await_within_ttft(
+                                iterator.__anext__(),
+                                deadline=ttft_deadline,
+                                provider=adapter.name,
+                                ttft_timeout_s=ttft_s,
+                            )
                     except StopAsyncIteration:
                         break
             except AdapterError as exc:
@@ -3609,8 +3677,28 @@ class FallbackEngine:
                     # message — so the hop is recorded as ``stream-truncated``
                     # and the empty-response log is skipped. The adapter has
                     # already emitted ``stream-truncation-detected``.
-                    truncated = isinstance(exc, StreamTruncatedError)
-                    if truncated:
+                    #
+                    # v2.17.0: a TTFT cut-off lands here too, and its buffer
+                    # is just as unterminated (opening events, no
+                    # ``message_stop``). Treat it like a truncation so the
+                    # chain-exhausted flush below never replays it as a
+                    # "complete empty answer" — that would hand the client a
+                    # 200 stream that never ends. Close the abandoned upstream
+                    # stream now rather than waiting for GC.
+                    ttft_cut = isinstance(exc, TTFTTimeoutError)
+                    truncated = isinstance(exc, StreamTruncatedError) or ttft_cut
+                    if ttft_cut:
+                        aclose_iter = getattr(event_iter, "aclose", None)
+                        if aclose_iter is not None:
+                            with contextlib.suppress(Exception):
+                                await aclose_iter()
+                        trace.record_failure(
+                            adapter.name,
+                            REASON_TTFT_TIMEOUT,
+                            detail=describe_adapter_error(exc),
+                            stream=True,
+                        )
+                    elif truncated:
                         trace.record_failure(
                             adapter.name,
                             REASON_STREAM_TRUNCATED,

{coderouter_cli-2.16.0 → coderouter_cli-2.17.0}/coderouter/routing/fallback_trace.py

@@ -111,6 +111,8 @@ REASON_EMPTY_STREAM: Final = "empty-stream"
 # ``connection`` (the transport itself broke). Produced only when the profile
 # sets ``stream_truncation_action: error``.
 REASON_STREAM_TRUNCATED: Final = "stream-truncated"
+# v2.17.0: no first real output within the provider's ``ttft_timeout_s``.
+REASON_TTFT_TIMEOUT: Final = "ttft-timeout"
 
 # -- degrade reasons (pre-existing strings; never produce a hop) -----------
 # Kept here so the whole ``reason`` vocabulary is readable in one place.
@@ -141,6 +143,7 @@ ATTEMPT_FAILURE_REASONS: Final[frozenset[str]] = frozenset(
         REASON_EMPTY_RESPONSE,
         REASON_EMPTY_STREAM,
         REASON_STREAM_TRUNCATED,
+        REASON_TTFT_TIMEOUT,
     }
 )
 
@@ -207,6 +210,8 @@ def describe_adapter_error(exc: AdapterError) -> str:
         # Not a transport fault — the HTTP body ended cleanly; it was the
         # LLM protocol inside it that never closed.
         return "no-terminator"
+    if reason == REASON_TTFT_TIMEOUT:
+        return "no-first-token"
     return f"transport={reason}"
 
 
@@ -227,10 +232,12 @@ def classify_adapter_error(exc: AdapterError) -> str:
     ``adapters.base`` is a TYPE_CHECKING-only import at module scope here
     (the runtime import order would otherwise cycle through the registry).
     """
-    from coderouter.adapters.base import StreamTruncatedError
+    from coderouter.adapters.base import StreamTruncatedError, TTFTTimeoutError
 
     if isinstance(exc, StreamTruncatedError):
         return REASON_STREAM_TRUNCATED
+    if isinstance(exc, TTFTTimeoutError):
+        return REASON_TTFT_TIMEOUT
     status = exc.status_code
     if status is None:
         message = str(exc).lower()

{coderouter_cli-2.16.0 → coderouter_cli-2.17.0}/docs/README.en.md

@@ -90,6 +90,7 @@ How CodeRouter works and its reliability mechanisms.
 - **partial-stitch** — Mid-stream partial stitching (v2.0-H) · [日本語](concepts/partial-stitch.md) · [English](concepts/partial-stitch.en.md)
 - **continuous-probing** — Continuous probing (v2.0-I) · [日本語](concepts/continuous-probing.md) · [English](concepts/continuous-probing.en.md)
 - **stream-truncation** — Stream truncation detection (v2.15.0) · [日本語](concepts/stream-truncation.md) · [English](concepts/stream-truncation.en.md)
+- **commit-point** — Pre-content failures, TTFT cut-off and chain-failure errors (v2.16.0/v2.17.0) · [日本語](concepts/commit-point.md) · [English](concepts/commit-point.en.md)
 - **low-memory-integration** — Integration guide for the proactive memory-budget guard on low-memory (8–16GB) hosts — VRAM/RAM detection, GGUF header introspection, and KV-cache-aware pre-dispatch fit decisions (`off`/`warn`/`fit`). **Implemented (v2.5.3)** · [日本語](low-memory-integration.md) · [English](low-memory-integration.en.md)
 
 ## 5. Design docs & records

{coderouter_cli-2.16.0 → coderouter_cli-2.17.0}/docs/README.md

@@ -95,6 +95,7 @@ CodeRouter の仕組みと信頼性機構。 / How CodeRouter works and its reli
 - **partial-stitch** — ストリーム途中の部分ステッチ (v2.0-H) / Mid-stream partial stitching · [日本語](concepts/partial-stitch.md) · [English](concepts/partial-stitch.en.md)
 - **continuous-probing** — 継続プロービング (v2.0-I) / Continuous probing · [日本語](concepts/continuous-probing.md) · [English](concepts/continuous-probing.en.md)
 - **stream-truncation** — ストリーム断絶検知 (v2.15.0) / Stream truncation detection · [日本語](concepts/stream-truncation.md) · [English](concepts/stream-truncation.en.md)
+- **commit-point** — 本文送出前の失敗の扱い・TTFT カットオフ・全滅時のエラー (v2.16.0/v2.17.0) / Pre-content failures, TTFT cut-off, chain-failure errors · [日本語](concepts/commit-point.md) · [English](concepts/commit-point.en.md)
 - **low-memory-integration** — 低メモリ機(8–16GB)向けメモリ予算ガードの統合ガイド。VRAM/RAM 検出・GGUFヘッダ解析・KVキャッシュ試算による事前フィット判定(`off`/`warn`/`fit`)。**実装済み (v2.5.3)** / Integration guide for the proactive memory-budget guard on low-memory (8–16GB) hosts — VRAM/RAM detection, GGUF header introspection, and KV-cache-aware pre-dispatch fit decisions (`off`/`warn`/`fit`). **Implemented (v2.5.3)** · [日本語](low-memory-integration.md) · [English](low-memory-integration.en.md)
 
 ## 5. 設計資料・記録 / Design docs & records

coderouter_cli-2.17.0/docs/concepts/commit-point.en.md

@@ -0,0 +1,68 @@
+# The commit point — which failures before the body ships can be saved
+
+> Applies to v2.16.0+ (`ttft_timeout_s` and `x-should-retry` are v2.17.0)
+> 日本語: [commit-point.md](./commit-point.md)
+
+A streaming response has a point of no return. Once even one byte of the body has reached the client, an upstream failure cannot be handed to another provider — the text already delivered and the new provider's text would not join up. CodeRouter calls that point the **commit point** and tries to send every failure that happens before it to the next provider.
+
+## What can happen before the commit point, and what CodeRouter does
+
+| Situation | Handling | Needs |
+|---|---|---|
+| Upstream refuses, times out, or answers 4xx/5xx (before the first event) | Next provider. If the whole chain fails: **an Anthropic-shaped error with a real HTTP status** (v2.16.0) | nothing |
+| Upstream never produces its first token (model loading, long prompt) | Next provider once `ttft_timeout_s` expires (v2.17.0) | `ttft_timeout_s` on the provider |
+| Opening events (`message_start` …) arrived but the stream ends or breaks before any content | Withhold the opening events from the client and move to the next provider | `empty_response_action: fallback` on the profile |
+| The stream breaks **after** content shipped | **No switch.** Detect and report it (`stream_truncation_action`, v2.15.0), or close with the content received (`partial_stitch_action: surface`) | those settings |
+
+CodeRouter cannot save the last row either. As far as we checked in 2026-10, LiteLLM and Bifrost do not switch after the body has shipped either.
+
+## `empty_response_action: fallback` is the effective commit point
+
+With this setting the engine holds back `message_start` and the other opening events and releases them together with the **first real output** (a text / tool_use / thinking delta). That moment is the commit point; anything that fails before it is moved to the next provider without the client seeing it.
+
+Turn it on first when serving Claude Code:
+
+```yaml
+profiles:
+  - name: claude-code
+    providers: [local-llamacpp, openrouter-free]
+    empty_response_action: fallback
+```
+
+## `ttft_timeout_s` — a deadline for the first token (v2.17.0)
+
+```yaml
+providers:
+  - name: local-llamacpp
+    kind: anthropic                  # pass straight through to llama-server's /v1/messages
+    base_url: http://127.0.0.1:8080
+    model: ""
+    ttft_timeout_s: 20     # no first output within 20 s → next provider
+```
+
+- **Starts** when the attempt on that provider starts.
+- **Stops** at the first real output; opening events alone do not stop it. Watching past the opening events needs `empty_response_action: fallback` (without it those events go to the client at once and the stream is committed).
+- **Never applied to the last provider of a chain** — there is nothing to fall back to, so waiting is the better answer.
+- Not applied to non-streaming requests (`timeout_s` bounds those).
+- **Only the `/v1/messages` (Anthropic) streaming path** is covered; `/v1/chat/completions` is not.
+- **Not applied to `kind: openai_compat` requests that carry tools** (Claude Code sends tools on almost every request). That combination calls the upstream non-streaming so tool calls can be repaired (the tool-turn downgrade), which makes the "first output" the whole answer; a deadline there would throw away slow but healthy local replies. To use the TTFT cut-off with a local llama.cpp / Ollama, connect with `kind: anthropic` (native `/v1/messages`) as in the example above. Note that CodeRouter's tool-call repair does not run on that route (repair lives on the OpenAI-translation path), so it suits models whose native tool calls are already well-formed.
+- An abandoned attempt is an ordinary failure: `X-CodeRouter-Fallback-Reason: ttft-timeout`, and the self-healing guards learn from it.
+
+Typical use: a local llama-server is still loading a model or chewing through a long prompt while a cloud provider could answer now. Start at two to three times your usual local TTFT; too short and every request escapes to the cloud.
+
+## What a failed chain returns (v2.16.0 / v2.17.0)
+
+| Upstream situation | HTTP | `error.type` | `x-should-retry` |
+|---|---|---|---|
+| Context overflow (any provider) | 400 | `invalid_request_error` (`prompt is too long: …`) | `false` |
+| A 400 / 422 that retrying cannot fix | 400 | `invalid_request_error` (upstream wording) | `false` |
+| 413 | 413 | `request_too_large` | `false` |
+| 429 | 429 | `rate_limit_error` (upstream `retry-after` passed through) | `true` |
+| Anything else (5xx, timeouts, 401/403, …) | 502 | `api_error` | `true` if at least one failure was retryable |
+
+A streaming request whose chain fails before the first event gets this HTTP status too, not `200` with an in-stream `error` event. Claude Code compacts on `prompt is too long` and uses `x-should-retry` / `retry-after` to decide whether to retry.
+
+## Known limits
+
+- **Nothing reaches the client while opening events are withheld.** If a long local prompt keeps that window open, the client's idle timeout (Claude Code's `CLAUDE_STREAM_IDLE_TIMEOUT_MS`) can fire. Keep `ttft_timeout_s` below it.
+- There is no keep-alive `ping` during that window yet.

coderouter_cli-2.17.0/docs/concepts/commit-point.md

@@ -0,0 +1,70 @@
+# コミット点 — 本文を送り出す前の失敗を、どこまで救えるか
+
+> 対象: v2.16.0 以降（`ttft_timeout_s` と `x-should-retry` は v2.17.0）
+> English: [commit-point.en.md](./commit-point.en.md)
+
+ストリーミング応答には「ここから先は取り消せない」という点があります。クライアントへ 1 バイトでも本文を送った後に上流が倒れると、別の provider に切り替えることはできません（途中まで届いた文章と、新しい provider の文章がつながらないため）。CodeRouter はこの点を **コミット点** と呼び、その手前で起きた失敗はできるだけ次の provider に回します。
+
+## コミット点の手前で起きることと、その扱い
+
+| 状況 | 扱い | 必要な設定 |
+|---|---|---|
+| 上流が接続を拒否・タイムアウト・4xx/5xx（最初のイベントより前） | 次の provider へ。全滅なら **HTTP ステータス付きの Anthropic 形式エラー**（v2.16.0） | なし |
+| 上流がいつまでも最初のトークンを返さない（モデル読み込み中、長いプロンプト処理） | `ttft_timeout_s` を過ぎたら次の provider へ（v2.17.0） | provider に `ttft_timeout_s` |
+| `message_start` などの前置イベントは来たが、本文が来ないまま切れる／空で終わる | 前置イベントをクライアントに見せずに保留し、次の provider へ | profile に `empty_response_action: fallback` |
+| 本文を送り出した**後**に切れる | **切り替えない**。検知して通知する（`stream_truncation_action`、v2.15.0）、または受け取り済みの本文で閉じる（`partial_stitch_action: surface`） | 各設定 |
+
+最後の行は CodeRouter でも救えません。2026-10 時点で確認した範囲では、LiteLLM や Bifrost も本文送出後には切り替えません。
+
+## `empty_response_action: fallback` が実質的なコミット点
+
+この設定にすると、エンジンは `message_start` などの前置イベントを手元に保留し、**最初の本物の出力**（text / tool_use / thinking の delta）が来た時点でまとめて送り出します。そこがコミット点です。それまでに起きた失敗は、クライアントから見えないまま次の provider に回ります。
+
+Claude Code で使うなら、まずこれを有効にしてください。
+
+```yaml
+profiles:
+  - name: claude-code
+    providers: [local-llamacpp, openrouter-free]
+    empty_response_action: fallback
+```
+
+## `ttft_timeout_s` — 最初のトークンの期限（v2.17.0）
+
+```yaml
+providers:
+  - name: local-llamacpp
+    kind: anthropic                  # llama-server の /v1/messages にそのまま流す
+    base_url: http://127.0.0.1:8080
+    model: ""
+    ttft_timeout_s: 20     # 20 秒以内に最初の出力が無ければ次の provider へ
+```
+
+- **数え始め**: その provider への試行を始めた時点。
+- **止まる条件**: 最初の本物の出力が来たとき。前置イベント（`message_start`）だけでは止まりません。ただし、前置イベントが来た後も期限を見続けるには `empty_response_action: fallback` が必要です（それが無いと前置イベントはすぐクライアントに送られ、コミット済みになるため）。
+- **チェーンの最後の provider には適用しません**。次が無いので、待つ方がましだからです。
+- 非ストリーミングの要求には適用しません（そちらは `timeout_s` が上限）。
+- **`/v1/messages`（Anthropic 形式）のストリーミングだけが対象**です。`/v1/chat/completions` には効きません。
+- **`kind: openai_compat` で tools 付きの要求（Claude Code はほぼ毎回 tools を送ります）には効きません。** この組み合わせは、tool call を修復するために上流を非ストリーミングで呼ぶ経路（tool-turn downgrade）を通るので、「最初の出力」が応答全体になってしまうからです。ここに期限をかけると、ゆっくりだが正常なローカルの応答を途中で捨てることになります。ローカルの llama.cpp / Ollama で TTFT カットオフを使うなら、上の例のように `kind: anthropic`（ネイティブの `/v1/messages`）で接続してください。ただしこの接続では、CodeRouter の tool call 修復は働きません（修復は OpenAI 形式から翻訳する経路でだけ動きます）。ネイティブで tool call を正しく出せるモデル向けの選択肢です。
+- 打ち切った試行は通常の失敗として扱われます。理由は `X-CodeRouter-Fallback-Reason: ttft-timeout`、self-healing のガードもこれを学習します。
+
+典型的な使い方: ローカルの llama-server がモデルを読み込み中、または長いプロンプトを処理中で、クラウドなら今すぐ答えられる、という場面です。値は「ふだんのローカルの TTFT の 2〜3 倍」程度から始めてください。短すぎると、毎回クラウドに逃げることになります。
+
+## 全滅したときに返すもの（v2.16.0 / v2.17.0）
+
+チェーン全体が失敗したときは、Anthropic API と同じ形のエラーを返します。
+
+| 上流の状況 | HTTP | `error.type` | `x-should-retry` |
+|---|---|---|---|
+| 文脈超過（どの provider でも） | 400 | `invalid_request_error`（`prompt is too long: …`） | `false` |
+| 再試行しても無駄な 400 / 422 | 400 | `invalid_request_error`（上流の文言そのまま） | `false` |
+| 413 | 413 | `request_too_large` | `false` |
+| 429 | 429 | `rate_limit_error`（上流の `retry-after` を転送） | `true` |
+| その他（5xx、タイムアウト、401/403 など） | 502 | `api_error` | 再試行可能な失敗が 1 つでもあれば `true` |
+
+ストリーミングの要求でも、最初のイベントより前に全滅した場合はこの HTTP ステータスで返ります（`200` の中に `error` イベントを入れる形ではありません）。Claude Code は `prompt is too long` を見て会話を要約し、`x-should-retry` / `retry-after` を見て再試行するかを決めます。
+
+## 既知の制限
+
+- **前置イベントを保留している間は、クライアントに何も送れません。** ローカルの長いプロンプト処理でこの時間が長いと、クライアント側の無通信タイムアウト（Claude Code の `CLAUDE_STREAM_IDLE_TIMEOUT_MS`）に当たる可能性があります。`ttft_timeout_s` をその値より短くしてください。
+- 保留中に `ping` を送って接続を保つ機能はまだありません。

{coderouter_cli-2.16.0 → coderouter_cli-2.17.0}/pyproject.toml

@@ -11,7 +11,7 @@
 # in plan.md §11.B; once granted, this name will become an alias and
 # `coderouter` will become the canonical distribution name.
 name = "coderouter-cli"
-version = "2.16.0"
+version = "2.17.0"
 description = "Local-first, free-first, fallback-built-in LLM router. Claude Code / OpenAI compatible."
 readme = "README.md"
 requires-python = ">=3.12"

{coderouter_cli-2.16.0 → coderouter_cli-2.17.0}/tests/test_anthropic_errors.py

@@ -131,7 +131,7 @@ def test_429_maps_to_rate_limit_with_retry_after() -> None:
         NoProvidersAvailableError("default", [_err(429, '{"error":{"message":"slow down"}}', retry_after="7")])
     )
     assert (mapped.status_code, mapped.error_type) == (429, "rate_limit_error")
-    assert mapped.headers == {"retry-after": "7"}
+    assert mapped.headers == {"retry-after": "7", "x-should-retry": "true"}
     assert mapped.message == "slow down"
 
 
@@ -288,6 +288,7 @@ def test_reject_raised_inside_engine_is_400_not_500(monkeypatch: pytest.MonkeyPa
     resp = TestClient(app).post("/v1/messages", json={**_BODY, "stream": stream})
     assert resp.status_code == 400, resp.text
     assert resp.json()["error"]["message"].startswith("prompt is too long")
+    assert resp.headers["x-should-retry"] == "false"
 
 
 @pytest.mark.asyncio
@@ -350,3 +351,31 @@ async def test_disconnect_watcher_quiet_when_client_stays() -> None:
     await asyncio.sleep(0.05)
     assert not watcher.fired
     await watcher.stop()
+# v2.17.0: x-should-retry
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    ("errors", "expected"),
+    [
+        ([_err(400, _LLAMACPP_OVERFLOW, retryable=False)], "false"),
+        ([_err(400, _ANTHROPIC_400, retryable=False)], "false"),
+        ([_err(413, "big", retryable=False)], "false"),
+        ([_err(429, "slow")], "true"),
+        ([_err(503, "busy")], "true"),
+        ([_err(401, "bad key", retryable=False), _err(403, "nope", retryable=False)], "false"),
+        ([_err(401, "bad key", retryable=False), _err(503, "busy")], "true"),
+        ([], "false"),
+    ],
+)
+def test_x_should_retry(errors: list[AdapterError], expected: str) -> None:
+    mapped = map_no_providers_error(NoProvidersAvailableError("default", errors))
+    assert mapped.headers["x-should-retry"] == expected
+
+
+@pytest.mark.parametrize("stream", [False, True])
+def test_x_should_retry_reaches_client(make_client, stream: bool) -> None:  # type: ignore[no-untyped-def]
+    client = make_client([_err(401, "bad key", retryable=False)])
+    resp = client.post("/v1/messages", json={**_BODY, "stream": stream})
+    assert resp.status_code == 502
+    assert resp.headers["x-should-retry"] == "false"