coderouter-cli 2.14.0__py3-none-any.whl → 2.15.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- coderouter/adapters/anthropic_native.py +106 -1
- coderouter/adapters/base.py +65 -0
- coderouter/adapters/openai_compat.py +60 -0
- coderouter/config/schemas.py +65 -0
- coderouter/doctor.py +48 -1
- coderouter/ingress/anthropic_routes.py +82 -12
- coderouter/ingress/openai_routes.py +31 -2
- coderouter/logging.py +143 -0
- coderouter/metrics/collector.py +40 -0
- coderouter/metrics/prometheus.py +46 -0
- coderouter/routing/__init__.py +8 -0
- coderouter/routing/fallback.py +304 -4
- coderouter/routing/fallback_trace.py +501 -0
- coderouter/state/audit_log.py +4 -0
- coderouter/vscode_init.py +10 -3
- {coderouter_cli-2.14.0.dist-info → coderouter_cli-2.15.1.dist-info}/METADATA +17 -5
- {coderouter_cli-2.14.0.dist-info → coderouter_cli-2.15.1.dist-info}/RECORD +20 -19
- {coderouter_cli-2.14.0.dist-info → coderouter_cli-2.15.1.dist-info}/WHEEL +1 -1
- {coderouter_cli-2.14.0.dist-info → coderouter_cli-2.15.1.dist-info}/entry_points.txt +0 -0
- {coderouter_cli-2.14.0.dist-info → coderouter_cli-2.15.1.dist-info}/licenses/LICENSE +0 -0
|
@@ -44,9 +44,14 @@ from coderouter.adapters.base import (
|
|
|
44
44
|
ChatResponse,
|
|
45
45
|
ProviderCallOverrides,
|
|
46
46
|
StreamChunk,
|
|
47
|
+
StreamTruncatedError,
|
|
47
48
|
)
|
|
48
49
|
from coderouter.credentials import resolve_provider_credential
|
|
49
|
-
from coderouter.logging import
|
|
50
|
+
from coderouter.logging import (
|
|
51
|
+
get_logger,
|
|
52
|
+
log_output_filter_applied,
|
|
53
|
+
log_stream_truncation_detected,
|
|
54
|
+
)
|
|
50
55
|
from coderouter.output_filters import OutputFilterChain
|
|
51
56
|
from coderouter.translation.anthropic import (
|
|
52
57
|
AnthropicRequest,
|
|
@@ -74,6 +79,74 @@ _RETRYABLE_STATUSES = {404, 408, 425, 429, 500, 502, 503, 504}
|
|
|
74
79
|
_DEFAULT_ANTHROPIC_VERSION = "2023-06-01"
|
|
75
80
|
|
|
76
81
|
|
|
82
|
+
class _AnthropicTerminatorTracker:
|
|
83
|
+
"""v2.15.0: watch an Anthropic SSE event stream for its terminator.
|
|
84
|
+
|
|
85
|
+
The parse loop in :meth:`AnthropicAdapter.stream_anthropic` used to be
|
|
86
|
+
stateless across events — ``current_event`` was reset on every blank
|
|
87
|
+
line and nothing remembered whether the stream had ended *properly*.
|
|
88
|
+
That made "upstream sent 30 deltas and hung up" indistinguishable from
|
|
89
|
+
"upstream sent the whole message". This tracker is the missing memory.
|
|
90
|
+
|
|
91
|
+
What counts as a terminator (deliberately generous, to keep the false
|
|
92
|
+
positive rate down — see ``stream_truncation_action`` in
|
|
93
|
+
``config/schemas.py``):
|
|
94
|
+
|
|
95
|
+
* ``message_stop`` — the canonical Anthropic terminator.
|
|
96
|
+
* ``message_delta`` carrying a non-null ``stop_reason`` — the message is
|
|
97
|
+
semantically complete even if the provider omits ``message_stop``.
|
|
98
|
+
Some Anthropic-compatible local servers do exactly that, and their
|
|
99
|
+
content is *not* truncated, so flagging them would be a false positive.
|
|
100
|
+
* a top-level ``error`` event — the upstream ended the stream on purpose
|
|
101
|
+
and the frame is forwarded to the client verbatim (current behavior).
|
|
102
|
+
Reporting truncation on top would double-report one failure.
|
|
103
|
+
|
|
104
|
+
``open_block_type`` tracks the currently-open content block so a stream
|
|
105
|
+
cut in the middle of a ``tool_use`` block's ``input_json_delta`` can be
|
|
106
|
+
reported as such: that is the case where the translation layer's
|
|
107
|
+
terminator synthesis closes a block whose argument JSON is provably
|
|
108
|
+
incomplete.
|
|
109
|
+
"""
|
|
110
|
+
|
|
111
|
+
__slots__ = ("events", "open_block_type", "saw_start", "saw_terminator")
|
|
112
|
+
|
|
113
|
+
def __init__(self) -> None:
|
|
114
|
+
"""Start with nothing observed."""
|
|
115
|
+
self.saw_terminator: bool = False
|
|
116
|
+
self.saw_start: bool = False
|
|
117
|
+
self.events: int = 0
|
|
118
|
+
self.open_block_type: str | None = None
|
|
119
|
+
|
|
120
|
+
def observe(self, event_type: str, data: Any) -> None:
|
|
121
|
+
"""Fold one parsed SSE event into the tracked state.
|
|
122
|
+
|
|
123
|
+
``data`` is whatever ``json.loads`` produced for the block, so it is
|
|
124
|
+
not guaranteed to be a mapping; non-dict payloads are treated as
|
|
125
|
+
empty rather than raising inside a streaming hot path.
|
|
126
|
+
"""
|
|
127
|
+
self.events += 1
|
|
128
|
+
if not isinstance(data, dict):
|
|
129
|
+
data = {}
|
|
130
|
+
if event_type == "message_start":
|
|
131
|
+
self.saw_start = True
|
|
132
|
+
elif event_type == "content_block_start":
|
|
133
|
+
block = data.get("content_block")
|
|
134
|
+
self.open_block_type = block.get("type") if isinstance(block, dict) else None
|
|
135
|
+
elif event_type == "content_block_stop":
|
|
136
|
+
self.open_block_type = None
|
|
137
|
+
elif event_type == "message_delta":
|
|
138
|
+
delta = data.get("delta")
|
|
139
|
+
if isinstance(delta, dict) and delta.get("stop_reason"):
|
|
140
|
+
self.saw_terminator = True
|
|
141
|
+
elif event_type in {"message_stop", "error"}:
|
|
142
|
+
self.saw_terminator = True
|
|
143
|
+
|
|
144
|
+
@property
|
|
145
|
+
def tool_call_in_flight(self) -> bool:
|
|
146
|
+
"""True when a ``tool_use`` block was still open at end-of-stream."""
|
|
147
|
+
return self.open_block_type == "tool_use"
|
|
148
|
+
|
|
149
|
+
|
|
77
150
|
def anthropic_messages_url(base_url: str) -> str:
|
|
78
151
|
"""Resolve the ``/v1/messages`` endpoint URL from a ``base_url``.
|
|
79
152
|
|
|
@@ -475,6 +548,12 @@ class AnthropicAdapter(BaseAdapter):
|
|
|
475
548
|
filter_chains: dict[int, OutputFilterChain] = {}
|
|
476
549
|
logged_flag: list[bool] = [False]
|
|
477
550
|
|
|
551
|
+
# v2.15.0 (stream-truncation): remember whether the upstream ever
|
|
552
|
+
# terminated its message. Resolved once per call — the knob is a
|
|
553
|
+
# profile-level setting threaded in through ProviderCallOverrides.
|
|
554
|
+
truncation_action = self.effective_stream_truncation_action(overrides)
|
|
555
|
+
terminator = _AnthropicTerminatorTracker()
|
|
556
|
+
|
|
478
557
|
try:
|
|
479
558
|
# H3: stream over the shared client (pool / keep-alive / TLS
|
|
480
559
|
# reuse). Only the ``stream(...)`` context is scoped here; the
|
|
@@ -514,6 +593,7 @@ class AnthropicAdapter(BaseAdapter):
|
|
|
514
593
|
current_event = None
|
|
515
594
|
data_lines = []
|
|
516
595
|
continue
|
|
596
|
+
terminator.observe(current_event, data_obj)
|
|
517
597
|
for out_event in self._process_stream_event_for_filters(
|
|
518
598
|
AnthropicStreamEvent(type=current_event, data=data_obj),
|
|
519
599
|
chains=filter_chains,
|
|
@@ -542,12 +622,37 @@ class AnthropicAdapter(BaseAdapter):
|
|
|
542
622
|
except json.JSONDecodeError:
|
|
543
623
|
data_obj = None
|
|
544
624
|
if data_obj is not None:
|
|
625
|
+
terminator.observe(current_event, data_obj)
|
|
545
626
|
for out_event in self._process_stream_event_for_filters(
|
|
546
627
|
AnthropicStreamEvent(type=current_event, data=data_obj),
|
|
547
628
|
chains=filter_chains,
|
|
548
629
|
logged_flag=logged_flag,
|
|
549
630
|
):
|
|
550
631
|
yield out_event
|
|
632
|
+
|
|
633
|
+
# v2.15.0 (stream-truncation): the upstream line iterator is
|
|
634
|
+
# exhausted. Reaching here with no terminator observed means the
|
|
635
|
+
# HTTP body ended cleanly while the Anthropic message was still
|
|
636
|
+
# open — the exact case transport-level error handling below
|
|
637
|
+
# cannot see (``httpx.RemoteProtocolError`` and timeouts already
|
|
638
|
+
# raise). Under ``off`` this block is inert, which is what keeps
|
|
639
|
+
# the default byte-for-byte identical to v2.14.0.
|
|
640
|
+
if truncation_action != "off" and not terminator.saw_terminator:
|
|
641
|
+
log_stream_truncation_detected(
|
|
642
|
+
logger,
|
|
643
|
+
self.name,
|
|
644
|
+
action=truncation_action,
|
|
645
|
+
wire="anthropic",
|
|
646
|
+
events_forwarded=terminator.events,
|
|
647
|
+
saw_stream_start=terminator.saw_start,
|
|
648
|
+
tool_call_in_flight=terminator.tool_call_in_flight,
|
|
649
|
+
)
|
|
650
|
+
if truncation_action == "error":
|
|
651
|
+
raise StreamTruncatedError(
|
|
652
|
+
"upstream stream truncated (no message_stop terminator)",
|
|
653
|
+
provider=self.name,
|
|
654
|
+
tool_call_in_flight=terminator.tool_call_in_flight,
|
|
655
|
+
)
|
|
551
656
|
except httpx.TimeoutException as exc:
|
|
552
657
|
raise AdapterError(
|
|
553
658
|
f"timeout streaming from {url}", provider=self.name, retryable=True
|
coderouter/adapters/base.py
CHANGED
|
@@ -137,6 +137,49 @@ class AdapterError(CodeRouterError):
|
|
|
137
137
|
return f"[{self.provider}{sc}] {super().__str__()}"
|
|
138
138
|
|
|
139
139
|
|
|
140
|
+
# v2.15.0 (stream-truncation): raised by an adapter when the upstream SSE
|
|
141
|
+
# stream ended *without* its protocol terminator (``message_stop`` on the
|
|
142
|
+
# Anthropic wire, ``data: [DONE]`` / a ``finish_reason`` on the OpenAI wire).
|
|
143
|
+
#
|
|
144
|
+
# HTTP-level breakage (timeout, ``httpx.RemoteProtocolError``) already became a
|
|
145
|
+
# plain ``AdapterError`` in both adapters. This subclass covers the layer
|
|
146
|
+
# mismatch the transport cannot see: the HTTP body ended cleanly, but the *LLM
|
|
147
|
+
# protocol* carried inside it was still mid-message.
|
|
148
|
+
#
|
|
149
|
+
# It subclasses ``AdapterError`` on purpose — every existing engine branch
|
|
150
|
+
# (``except AdapterError``) keeps working untouched, and the L2/L4/L5/L6 guards
|
|
151
|
+
# learn from the failure automatically. The subclass only adds identity, so
|
|
152
|
+
# ``classify_adapter_error`` can label the hop ``stream-truncated`` instead of
|
|
153
|
+
# the generic ``upstream-error``.
|
|
154
|
+
#
|
|
155
|
+
# Raised only when the active profile sets ``stream_truncation_action: error``.
|
|
156
|
+
# Under ``off`` (the default) and ``warn`` the adapter returns normally and the
|
|
157
|
+
# legacy terminator-synthesis path in ``coderouter.translation.convert`` (H6 /
|
|
158
|
+
# M9) runs byte-for-byte as before.
|
|
159
|
+
class StreamTruncatedError(AdapterError):
|
|
160
|
+
"""Upstream SSE stream ended without its protocol terminator event."""
|
|
161
|
+
|
|
162
|
+
def __init__(
|
|
163
|
+
self,
|
|
164
|
+
message: str,
|
|
165
|
+
*,
|
|
166
|
+
provider: str,
|
|
167
|
+
tool_call_in_flight: bool = False,
|
|
168
|
+
) -> None:
|
|
169
|
+
"""Construct a truncation error.
|
|
170
|
+
|
|
171
|
+
Args:
|
|
172
|
+
message: Human-readable reason. Never contains upstream body text.
|
|
173
|
+
provider: The ``ProviderConfig.name`` whose stream was cut.
|
|
174
|
+
tool_call_in_flight: True when the stream was cut while a
|
|
175
|
+
``tool_use`` / ``tool_calls`` block was still open, i.e. the
|
|
176
|
+
accumulated argument JSON is certainly incomplete. Carried for
|
|
177
|
+
observability only — it never changes control flow.
|
|
178
|
+
"""
|
|
179
|
+
super().__init__(message, provider=provider, status_code=None, retryable=True)
|
|
180
|
+
self.tool_call_in_flight = tool_call_in_flight
|
|
181
|
+
|
|
182
|
+
|
|
140
183
|
# v0.6-B: per-call overrides resolved from the active profile. The engine
|
|
141
184
|
# builds one instance per request (since a profile is invariant across its
|
|
142
185
|
# chain) and threads it through every adapter call on that chain. Adapters
|
|
@@ -150,6 +193,13 @@ class AdapterError(CodeRouterError):
|
|
|
150
193
|
# - ``append_system_prompt=""`` is a meaningful explicit value: "for
|
|
151
194
|
# this profile, clear the provider's directive". The adapter must
|
|
152
195
|
# distinguish ``None`` (no override) from ``""`` (override-to-empty).
|
|
196
|
+
#
|
|
197
|
+
# v2.15.0 (stream-truncation): ``stream_truncation_action`` rides the same
|
|
198
|
+
# channel. It is a *profile*-level knob but has to be read inside the adapter's
|
|
199
|
+
# SSE parse loop, and ``ProviderCallOverrides`` is the only object the engine
|
|
200
|
+
# already threads into every adapter call. Its default of ``"off"`` keeps
|
|
201
|
+
# ``ProviderCallOverrides()`` (and every legacy call site that passes nothing)
|
|
202
|
+
# on the pre-v2.15.0 behavior.
|
|
153
203
|
class ProviderCallOverrides(BaseModel):
|
|
154
204
|
"""Per-call provider overrides, resolved from the active profile."""
|
|
155
205
|
|
|
@@ -157,6 +207,7 @@ class ProviderCallOverrides(BaseModel):
|
|
|
157
207
|
|
|
158
208
|
timeout_s: float | None = None
|
|
159
209
|
append_system_prompt: str | None = None
|
|
210
|
+
stream_truncation_action: Literal["off", "warn", "error"] = "off"
|
|
160
211
|
|
|
161
212
|
|
|
162
213
|
class BaseAdapter(ABC):
|
|
@@ -236,6 +287,20 @@ class BaseAdapter(ABC):
|
|
|
236
287
|
return overrides.append_system_prompt or None
|
|
237
288
|
return self.config.append_system_prompt
|
|
238
289
|
|
|
290
|
+
def effective_stream_truncation_action(
|
|
291
|
+
self, overrides: ProviderCallOverrides | None
|
|
292
|
+
) -> str:
|
|
293
|
+
"""v2.15.0: resolve ``stream_truncation_action`` for this call.
|
|
294
|
+
|
|
295
|
+
There is no per-provider counterpart — the knob lives on the profile
|
|
296
|
+
only — so this is just a null-safe read. ``None`` overrides (legacy
|
|
297
|
+
call sites, unit tests constructing adapters directly) resolve to
|
|
298
|
+
``"off"``, which is the pre-v2.15.0 behavior.
|
|
299
|
+
"""
|
|
300
|
+
if overrides is None:
|
|
301
|
+
return "off"
|
|
302
|
+
return overrides.stream_truncation_action
|
|
303
|
+
|
|
239
304
|
@abstractmethod
|
|
240
305
|
async def healthcheck(self) -> bool:
|
|
241
306
|
"""Lightweight check that the upstream is reachable. Return True if healthy."""
|
|
@@ -28,12 +28,14 @@ from coderouter.adapters.base import (
|
|
|
28
28
|
ChatResponse,
|
|
29
29
|
ProviderCallOverrides,
|
|
30
30
|
StreamChunk,
|
|
31
|
+
StreamTruncatedError,
|
|
31
32
|
)
|
|
32
33
|
from coderouter.credentials import resolve_provider_credential
|
|
33
34
|
from coderouter.logging import (
|
|
34
35
|
get_logger,
|
|
35
36
|
log_capability_degraded,
|
|
36
37
|
log_output_filter_applied,
|
|
38
|
+
log_stream_truncation_detected,
|
|
37
39
|
)
|
|
38
40
|
from coderouter.output_filters import OutputFilterChain
|
|
39
41
|
|
|
@@ -347,6 +349,20 @@ class OpenAICompatAdapter(BaseAdapter):
|
|
|
347
349
|
# Captured for the closing flush chunk (if any): reuse the last
|
|
348
350
|
# seen chunk's id/model so the flush emission looks native.
|
|
349
351
|
last_chunk_template: dict[str, Any] | None = None
|
|
352
|
+
|
|
353
|
+
# v2.15.0 (stream-truncation): the loop below used to treat "saw
|
|
354
|
+
# [DONE]" and "the line iterator ran out" as the same event. These
|
|
355
|
+
# three flags are the memory that tells them apart.
|
|
356
|
+
#
|
|
357
|
+
# A terminator is EITHER ``data: [DONE]`` OR a ``finish_reason`` on
|
|
358
|
+
# any choice. Accepting ``finish_reason`` matters: a provider that
|
|
359
|
+
# emits a complete message and then closes without the ``[DONE]``
|
|
360
|
+
# sentinel is not truncated, and flagging it would be a false
|
|
361
|
+
# positive. See ``stream_truncation_action`` in ``config/schemas.py``.
|
|
362
|
+
truncation_action = self.effective_stream_truncation_action(overrides)
|
|
363
|
+
saw_terminator = False
|
|
364
|
+
chunks_seen = 0
|
|
365
|
+
tool_call_in_flight = False
|
|
350
366
|
try:
|
|
351
367
|
# H3: stream over the shared client so the connection pool /
|
|
352
368
|
# keep-alive / TLS session are reused. Only the ``stream(...)``
|
|
@@ -376,11 +392,31 @@ class OpenAICompatAdapter(BaseAdapter):
|
|
|
376
392
|
continue
|
|
377
393
|
data_str = line[len("data:") :].strip()
|
|
378
394
|
if data_str == "[DONE]":
|
|
395
|
+
# v2.15.0: the sentinel — this stream ended on purpose.
|
|
396
|
+
saw_terminator = True
|
|
379
397
|
break
|
|
380
398
|
try:
|
|
381
399
|
payload_obj = json.loads(data_str)
|
|
382
400
|
except json.JSONDecodeError:
|
|
383
401
|
continue # skip malformed chunks rather than abort
|
|
402
|
+
# v2.15.0 (stream-truncation): fold this chunk into the
|
|
403
|
+
# terminator state before any filtering / validation, so a
|
|
404
|
+
# chunk that is later dropped as malformed still counts as
|
|
405
|
+
# "the upstream got this far".
|
|
406
|
+
chunks_seen += 1
|
|
407
|
+
for _choice in payload_obj.get("choices") or []:
|
|
408
|
+
if not isinstance(_choice, dict):
|
|
409
|
+
continue
|
|
410
|
+
if _choice.get("finish_reason"):
|
|
411
|
+
# A finish_reason means the message is complete
|
|
412
|
+
# even if the [DONE] sentinel never arrives, and
|
|
413
|
+
# it also closes any tool call that was open.
|
|
414
|
+
saw_terminator = True
|
|
415
|
+
tool_call_in_flight = False
|
|
416
|
+
elif isinstance(_choice.get("delta"), dict) and _choice[
|
|
417
|
+
"delta"
|
|
418
|
+
].get("tool_calls"):
|
|
419
|
+
tool_call_in_flight = True
|
|
384
420
|
if strip_reasoning:
|
|
385
421
|
stripped = _strip_reasoning_field(
|
|
386
422
|
payload_obj.get("choices"), delta_key=True
|
|
@@ -446,6 +482,30 @@ class OpenAICompatAdapter(BaseAdapter):
|
|
|
446
482
|
streaming=True,
|
|
447
483
|
)
|
|
448
484
|
output_filter_logged = True
|
|
485
|
+
|
|
486
|
+
# v2.15.0 (stream-truncation): decided AFTER the filter flush on
|
|
487
|
+
# purpose — the flush is the last chance to hand the caller every
|
|
488
|
+
# safe byte the filter chain was still holding, and under
|
|
489
|
+
# ``partial_stitch_action: surface`` those bytes end up in the
|
|
490
|
+
# partial content the client receives. Under ``off`` this block is
|
|
491
|
+
# inert, keeping the default byte-for-byte identical to v2.14.0.
|
|
492
|
+
if truncation_action != "off" and not saw_terminator:
|
|
493
|
+
log_stream_truncation_detected(
|
|
494
|
+
logger,
|
|
495
|
+
self.name,
|
|
496
|
+
action=truncation_action,
|
|
497
|
+
wire="openai",
|
|
498
|
+
events_forwarded=chunks_seen,
|
|
499
|
+
saw_stream_start=chunks_seen > 0,
|
|
500
|
+
tool_call_in_flight=tool_call_in_flight,
|
|
501
|
+
)
|
|
502
|
+
if truncation_action == "error":
|
|
503
|
+
raise StreamTruncatedError(
|
|
504
|
+
"upstream stream truncated "
|
|
505
|
+
"(no [DONE] sentinel and no finish_reason)",
|
|
506
|
+
provider=self.name,
|
|
507
|
+
tool_call_in_flight=tool_call_in_flight,
|
|
508
|
+
)
|
|
449
509
|
except httpx.TimeoutException as exc:
|
|
450
510
|
raise AdapterError(
|
|
451
511
|
f"timeout streaming from {url}", provider=self.name, retryable=True
|
coderouter/config/schemas.py
CHANGED
|
@@ -1232,6 +1232,71 @@ class FallbackChain(BaseModel):
|
|
|
1232
1232
|
),
|
|
1233
1233
|
)
|
|
1234
1234
|
|
|
1235
|
+
# --- v2.15.0 (stream-truncation): upstream SSE cut without a terminator --
|
|
1236
|
+
#
|
|
1237
|
+
# A streaming upstream can end its HTTP body *cleanly* while the LLM
|
|
1238
|
+
# protocol carried inside it is still mid-message: no ``message_stop`` on
|
|
1239
|
+
# the Anthropic wire, no ``data: [DONE]`` and no ``finish_reason`` on the
|
|
1240
|
+
# OpenAI wire. llama.cpp slot preemption, an ``--n-predict`` cut-off, a
|
|
1241
|
+
# front proxy closing an EOF-delimited body, or an OOM'd local server all
|
|
1242
|
+
# produce exactly this shape. Transport-level breakage (timeout,
|
|
1243
|
+
# ``httpx.RemoteProtocolError``) was already caught; this is the layer the
|
|
1244
|
+
# transport cannot see.
|
|
1245
|
+
#
|
|
1246
|
+
# Before v2.15.0 that stream was indistinguishable from a complete one:
|
|
1247
|
+
# the adapters never recorded whether a terminator arrived, and the
|
|
1248
|
+
# translation layer's terminator-synthesis guards (H6 / M9 in
|
|
1249
|
+
# ``coderouter.translation.convert``) then fabricated a ``stop_reason:
|
|
1250
|
+
# end_turn`` / ``finish_reason: "stop"`` so the client would not hang. The
|
|
1251
|
+
# synthesis is correct and stays; what was missing is telling the *engine*
|
|
1252
|
+
# that it happened.
|
|
1253
|
+
#
|
|
1254
|
+
# * ``off`` — no detection, no log, no metric. Backward-compatible
|
|
1255
|
+
# default: byte-for-byte identical to v2.14.0 on every path.
|
|
1256
|
+
# * ``warn`` — emit a ``stream-truncation-detected`` log line (and the
|
|
1257
|
+
# ``stream_truncated_total`` metric); the stream still
|
|
1258
|
+
# terminates through the legacy synthesis path, so the
|
|
1259
|
+
# client sees exactly what it saw before.
|
|
1260
|
+
# * ``error`` — the adapter raises ``StreamTruncatedError`` (a retryable
|
|
1261
|
+
# ``AdapterError``) at the point the terminator should have
|
|
1262
|
+
# arrived. The engine's existing branches take over: no
|
|
1263
|
+
# bytes forwarded yet → fall back to the next provider with
|
|
1264
|
+
# reason ``stream-truncated``; bytes already forwarded →
|
|
1265
|
+
# ``MidStreamError``, which ``partial_stitch_action:
|
|
1266
|
+
# surface`` renders as a graceful close carrying a
|
|
1267
|
+
# ``coderouter_partial`` event with reason
|
|
1268
|
+
# ``stream_truncated``.
|
|
1269
|
+
#
|
|
1270
|
+
# Caveats for ``error``:
|
|
1271
|
+
# * False positives are possible against an upstream that legitimately
|
|
1272
|
+
# omits the terminator. A ``message_delta`` carrying a ``stop_reason``
|
|
1273
|
+
# (Anthropic) and a ``finish_reason`` on any choice (OpenAI) are both
|
|
1274
|
+
# accepted as terminators to keep that risk low, but running ``warn``
|
|
1275
|
+
# first to measure the real rate is the recommended rollout.
|
|
1276
|
+
# * Falling back re-generates the answer on the next provider, so the
|
|
1277
|
+
# tokens the truncated attempt burned are paid twice.
|
|
1278
|
+
# * On the Anthropic streaming path the pre-content fallback requires
|
|
1279
|
+
# ``empty_response_action: fallback`` as well — that is the knob that
|
|
1280
|
+
# withholds the opening events from the client. Without it the opening
|
|
1281
|
+
# ``message_start`` has already shipped, so a truncation is by
|
|
1282
|
+
# definition mid-stream.
|
|
1283
|
+
stream_truncation_action: Literal["off", "warn", "error"] = Field(
|
|
1284
|
+
default="off",
|
|
1285
|
+
description=(
|
|
1286
|
+
"v2.15.0: action when an upstream SSE stream ends without its "
|
|
1287
|
+
"protocol terminator (no ``message_stop`` / no ``[DONE]`` and no "
|
|
1288
|
+
"``finish_reason``). ``off`` (default) does nothing — identical to "
|
|
1289
|
+
"v2.14.0. ``warn`` emits a ``stream-truncation-detected`` log and "
|
|
1290
|
+
"metric while the stream still ends through the legacy terminator "
|
|
1291
|
+
"synthesis. ``error`` raises a retryable ``StreamTruncatedError`` "
|
|
1292
|
+
"so the engine falls back to the next provider (reason "
|
|
1293
|
+
"``stream-truncated``) when nothing has reached the client yet, or "
|
|
1294
|
+
"raises ``MidStreamError`` when it has. Pair with "
|
|
1295
|
+
"``empty_response_action: fallback`` for pre-content fallback on "
|
|
1296
|
+
"the Anthropic streaming path."
|
|
1297
|
+
),
|
|
1298
|
+
)
|
|
1299
|
+
|
|
1235
1300
|
# --- ⑧ (empty-response): per-request empty-response fallback ------------
|
|
1236
1301
|
#
|
|
1237
1302
|
# Some local backends (observed: gemma4:26b on ``no_tool_temptation``
|
coderouter/doctor.py
CHANGED
|
@@ -635,6 +635,49 @@ _PROBE_TOOL_SPEC_ANTHROPIC = {
|
|
|
635
635
|
}
|
|
636
636
|
|
|
637
637
|
|
|
638
|
+
# v2.15.1: llama.cpp changed ``llama-server``'s default listen port from
|
|
639
|
+
# 8080 to 9931 (ggml-org/llama.cpp#30159, build b11521, 2026-10-09). A server
|
|
640
|
+
# started without ``--port`` on a new build is invisible to a provider that
|
|
641
|
+
# still points at :8080, and the reverse holds for configs written against
|
|
642
|
+
# the new default. When the basic probe cannot connect to one of those two
|
|
643
|
+
# ports on a loopback host, check the other one so the transport error can
|
|
644
|
+
# name the likely cause instead of looking like "server not running".
|
|
645
|
+
_LLAMACPP_PORT_PAIR: dict[int, int] = {8080: 9931, 9931: 8080}
|
|
646
|
+
_LOOPBACK_HOSTS = frozenset({"localhost", "127.0.0.1", "::1"})
|
|
647
|
+
|
|
648
|
+
|
|
649
|
+
async def _llamacpp_port_hint(base_url: object) -> str | None:
|
|
650
|
+
"""Return a hint when the other llama.cpp default port answers.
|
|
651
|
+
|
|
652
|
+
Only fires for loopback hosts on port 8080 or 9931. The alternate
|
|
653
|
+
port is probed at ``GET /health`` (llama-server's readiness route)
|
|
654
|
+
with a short timeout on the IPv4 literal — ``localhost`` can resolve
|
|
655
|
+
to ``::1`` first on macOS, where llama-server is not listening. The
|
|
656
|
+
config is never rewritten; the operator decides.
|
|
657
|
+
"""
|
|
658
|
+
try:
|
|
659
|
+
parsed = httpx.URL(str(base_url))
|
|
660
|
+
except (httpx.InvalidURL, TypeError):
|
|
661
|
+
return None
|
|
662
|
+
if parsed.host not in _LOOPBACK_HOSTS or parsed.port not in _LLAMACPP_PORT_PAIR:
|
|
663
|
+
return None
|
|
664
|
+
other = _LLAMACPP_PORT_PAIR[parsed.port]
|
|
665
|
+
try:
|
|
666
|
+
async with httpx.AsyncClient(timeout=1.5) as client:
|
|
667
|
+
resp = await client.get(f"http://127.0.0.1:{other}/health")
|
|
668
|
+
except httpx.HTTPError:
|
|
669
|
+
return None
|
|
670
|
+
if resp.status_code >= 500:
|
|
671
|
+
return None
|
|
672
|
+
return (
|
|
673
|
+
f"Something is answering on port {other} instead. llama.cpp "
|
|
674
|
+
f"builds b11521+ listen on 9931 when started without --port "
|
|
675
|
+
f"(older builds: 8080). Either start llama-server with "
|
|
676
|
+
f"--port {parsed.port}, or change this provider's base_url to "
|
|
677
|
+
f"port {other}"
|
|
678
|
+
)
|
|
679
|
+
|
|
680
|
+
|
|
638
681
|
async def _probe_auth_and_basic_chat(
|
|
639
682
|
provider: ProviderConfig,
|
|
640
683
|
) -> ProbeResult:
|
|
@@ -670,10 +713,14 @@ async def _probe_auth_and_basic_chat(
|
|
|
670
713
|
)
|
|
671
714
|
|
|
672
715
|
if status is None:
|
|
716
|
+
detail = f"could not reach {url}: {raw}"
|
|
717
|
+
hint = await _llamacpp_port_hint(provider.base_url)
|
|
718
|
+
if hint:
|
|
719
|
+
detail = f"{detail}. {hint}"
|
|
673
720
|
return ProbeResult(
|
|
674
721
|
name="auth+basic-chat",
|
|
675
722
|
verdict=ProbeVerdict.TRANSPORT_ERROR,
|
|
676
|
-
detail=
|
|
723
|
+
detail=detail,
|
|
677
724
|
)
|
|
678
725
|
|
|
679
726
|
if status in (401, 403):
|
|
@@ -30,6 +30,7 @@ from typing import Any
|
|
|
30
30
|
from fastapi import APIRouter, Header, HTTPException, Request
|
|
31
31
|
from fastapi.responses import JSONResponse, StreamingResponse
|
|
32
32
|
|
|
33
|
+
from coderouter.adapters.base import StreamTruncatedError
|
|
33
34
|
from coderouter.guards.tool_loop import ToolCountExceededError, ToolLoopBreakError
|
|
34
35
|
from coderouter.logging import get_logger
|
|
35
36
|
from coderouter.routing import (
|
|
@@ -38,6 +39,10 @@ from coderouter.routing import (
|
|
|
38
39
|
NoProvidersAvailableError,
|
|
39
40
|
)
|
|
40
41
|
from coderouter.routing.auto_router import RESERVED_PROFILE_NAME, classify
|
|
42
|
+
from coderouter.routing.fallback_trace import (
|
|
43
|
+
SSE_FALLBACK_EVENT,
|
|
44
|
+
current_fallback_trace,
|
|
45
|
+
)
|
|
41
46
|
from coderouter.token_estimation import extract_text_from_anthropic_request
|
|
42
47
|
from coderouter.token_estimation_accurate import count_tokens, is_accuracy_available
|
|
43
48
|
from coderouter.translation import (
|
|
@@ -54,6 +59,11 @@ _ANTHROPIC_VERSION_HEADER = "anthropic-version"
|
|
|
54
59
|
_ANTHROPIC_BETA_HEADER = "anthropic-beta"
|
|
55
60
|
_CTX_BUDGET_HEADER = "X-CodeRouter-Context-Budget"
|
|
56
61
|
_DRIFT_HEADER = "X-CodeRouter-Drift"
|
|
62
|
+
# v2.15.0: the ``X-CodeRouter-Fallback-*`` family is built by
|
|
63
|
+
# ``FallbackTrace.header_values()`` rather than named here, because the set
|
|
64
|
+
# of headers varies with the trace (``-To`` is absent when the whole chain
|
|
65
|
+
# was exhausted). The names themselves live next to the trace so the
|
|
66
|
+
# engine, the ingress and the tests all read one definition.
|
|
57
67
|
|
|
58
68
|
# M14: overall SSE stream ceiling. A single streamed request can legitimately
|
|
59
69
|
# run much longer than one provider call (long generations, tool turns), so we
|
|
@@ -258,6 +268,20 @@ async def messages(
|
|
|
258
268
|
drift_severity = engine.last_drift_severity
|
|
259
269
|
if drift_severity:
|
|
260
270
|
stream_headers[_DRIFT_HEADER] = drift_severity
|
|
271
|
+
# v2.15.0: fallback headers on the streaming path carry what is
|
|
272
|
+
# already known when the response commits — i.e. the pre-attempt
|
|
273
|
+
# hops (paid gate / budget / backend health / self-healing) that
|
|
274
|
+
# ``apply_context_budget`` recorded while resolving the chain just
|
|
275
|
+
# above. Runtime attempt failures happen *after* the HTTP headers
|
|
276
|
+
# have shipped and are therefore physically unreachable here; they
|
|
277
|
+
# are delivered instead as the trailing ``coderouter_fallback`` SSE
|
|
278
|
+
# metadata event emitted by ``_anthropic_sse_iterator`` (the same
|
|
279
|
+
# trailer mechanism v2.0-H uses for ``coderouter_partial``), and as
|
|
280
|
+
# ``fallback-occurred`` log lines. Same commit-order constraint the
|
|
281
|
+
# drift header comment above describes.
|
|
282
|
+
pre_dispatch_trace = current_fallback_trace()
|
|
283
|
+
if pre_dispatch_trace is not None:
|
|
284
|
+
stream_headers.update(pre_dispatch_trace.header_values())
|
|
261
285
|
# M14: wrap the SSE iterator with an overall timeout + client
|
|
262
286
|
# disconnect cleanup so a wedged upstream cannot pin the client
|
|
263
287
|
# (or the upstream socket) open forever.
|
|
@@ -309,6 +333,15 @@ async def messages(
|
|
|
309
333
|
resp_headers[_CTX_BUDGET_HEADER] = ctx_budget_status
|
|
310
334
|
if drift_severity:
|
|
311
335
|
resp_headers[_DRIFT_HEADER] = drift_severity
|
|
336
|
+
# v2.15.0: fallback reason headers. The engine wrote its hops onto the
|
|
337
|
+
# request-scoped trace while ``generate_anthropic`` was awaited above,
|
|
338
|
+
# so by here the trail is complete — ``header_values()`` returns an
|
|
339
|
+
# empty dict (and this whole block is inert) when the first provider
|
|
340
|
+
# served the request, which keeps the healthy path byte-identical to
|
|
341
|
+
# v2.14.0.
|
|
342
|
+
fallback_trace = current_fallback_trace()
|
|
343
|
+
if fallback_trace is not None:
|
|
344
|
+
resp_headers.update(fallback_trace.header_values())
|
|
312
345
|
|
|
313
346
|
if resp_headers:
|
|
314
347
|
return JSONResponse(
|
|
@@ -511,6 +544,17 @@ async def _anthropic_sse_iterator(
|
|
|
511
544
|
except (KeyError, ValueError):
|
|
512
545
|
pass
|
|
513
546
|
|
|
547
|
+
# v2.15.0 (stream-truncation): name the specific failure on the
|
|
548
|
+
# metadata event. ``mid_stream_failure`` stays the default so every
|
|
549
|
+
# pre-v2.15.0 case is byte-identical; ``stream_truncated`` tells the
|
|
550
|
+
# client the difference between "the provider errored out" and "the
|
|
551
|
+
# provider went quiet with the message still open".
|
|
552
|
+
partial_reason = (
|
|
553
|
+
"stream_truncated"
|
|
554
|
+
if isinstance(exc.original, StreamTruncatedError)
|
|
555
|
+
else "mid_stream_failure"
|
|
556
|
+
)
|
|
557
|
+
|
|
514
558
|
if partial_action == "surface" and exc.partial_content:
|
|
515
559
|
# Emit message_delta with accumulated usage (signals stream end).
|
|
516
560
|
yield _format_anthropic_sse(AnthropicStreamEvent(
|
|
@@ -533,21 +577,32 @@ async def _anthropic_sse_iterator(
|
|
|
533
577
|
"type": "coderouter_partial",
|
|
534
578
|
"partial_content": exc.partial_content,
|
|
535
579
|
"provider": exc.provider,
|
|
536
|
-
"reason":
|
|
580
|
+
"reason": partial_reason,
|
|
537
581
|
"original_error": str(exc.original)[:200],
|
|
538
582
|
},
|
|
539
583
|
))
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
584
|
+
# v2.15.0: keep the pre-v2.15.0 key set (and order) intact for
|
|
585
|
+
# every failure that is not a truncation. The JSON formatter
|
|
586
|
+
# walks ``record.__dict__`` and emits whatever ``extra`` carries,
|
|
587
|
+
# so an unconditional ``reason`` would change the shape of this
|
|
588
|
+
# log line even under the default ``stream_truncation_action:
|
|
589
|
+
# off`` — where no truncation can be detected in the first place
|
|
590
|
+
# and the release claims byte-for-byte compatibility with
|
|
591
|
+
# v2.14.0. The key is therefore appended only when it says
|
|
592
|
+
# something new.
|
|
593
|
+
_stitch_extra: dict[str, Any] = {
|
|
594
|
+
"provider": exc.provider,
|
|
595
|
+
"profile": profile_name,
|
|
596
|
+
"text_blocks": len(exc.partial_content),
|
|
597
|
+
"text_length": sum(
|
|
598
|
+
len(b.get("text", "")) for b in exc.partial_content
|
|
599
|
+
),
|
|
600
|
+
}
|
|
601
|
+
if partial_reason != "mid_stream_failure":
|
|
602
|
+
# Lets a dashboard split "provider errored" from "provider
|
|
603
|
+
# went quiet" without re-parsing the SSE.
|
|
604
|
+
_stitch_extra["reason"] = partial_reason
|
|
605
|
+
logger.info("partial-stitch-surfaced", extra=_stitch_extra)
|
|
551
606
|
else:
|
|
552
607
|
err_event = AnthropicStreamEvent(
|
|
553
608
|
type="error",
|
|
@@ -561,6 +616,21 @@ async def _anthropic_sse_iterator(
|
|
|
561
616
|
)
|
|
562
617
|
yield _format_anthropic_sse(err_event)
|
|
563
618
|
|
|
619
|
+
# v2.15.0: trailing fallback metadata. Emitted after the stream has
|
|
620
|
+
# terminated (normally or with an in-stream error frame), so it never
|
|
621
|
+
# interleaves with the Anthropic event sequence a client is parsing —
|
|
622
|
+
# exactly the position ``coderouter_partial`` occupies. Client-optional:
|
|
623
|
+
# spec-compliant Anthropic SDKs ignore unknown event types, and the
|
|
624
|
+
# frame is only produced when a fallback actually happened.
|
|
625
|
+
trace = current_fallback_trace()
|
|
626
|
+
if trace is not None and trace.occurred:
|
|
627
|
+
yield _format_anthropic_sse(
|
|
628
|
+
AnthropicStreamEvent(
|
|
629
|
+
type=SSE_FALLBACK_EVENT,
|
|
630
|
+
data=trace.as_event_payload(),
|
|
631
|
+
)
|
|
632
|
+
)
|
|
633
|
+
|
|
564
634
|
|
|
565
635
|
def _format_anthropic_sse(ev: AnthropicStreamEvent) -> str:
|
|
566
636
|
"""Serialize an :class:`AnthropicStreamEvent` onto the SSE wire.
|