coderouter-cli 2.13.0__py3-none-any.whl → 2.15.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- coderouter/adapters/anthropic_native.py +110 -3
- coderouter/adapters/base.py +65 -0
- coderouter/adapters/openai_compat.py +65 -2
- coderouter/cli.py +140 -2
- coderouter/config/loader.py +24 -1
- coderouter/config/schemas.py +209 -0
- coderouter/credentials.py +332 -0
- coderouter/doctor_apply.py +54 -9
- coderouter/ingress/anthropic_routes.py +82 -12
- coderouter/ingress/dashboard_routes.py +24 -4
- coderouter/ingress/metrics_routes.py +13 -2
- coderouter/ingress/openai_routes.py +31 -2
- coderouter/ingress/ui_auth.py +94 -0
- coderouter/logging.py +150 -0
- coderouter/metrics/collector.py +40 -0
- coderouter/metrics/prometheus.py +46 -0
- coderouter/rollback.py +217 -0
- coderouter/routing/__init__.py +8 -0
- coderouter/routing/fallback.py +304 -4
- coderouter/routing/fallback_trace.py +501 -0
- coderouter/secret_redaction.py +606 -0
- coderouter/state/audit_log.py +10 -0
- coderouter/state/request_log.py +6 -0
- coderouter/vscode_init.py +166 -23
- {coderouter_cli-2.13.0.dist-info → coderouter_cli-2.15.0.dist-info}/METADATA +2 -2
- {coderouter_cli-2.13.0.dist-info → coderouter_cli-2.15.0.dist-info}/RECORD +29 -24
- {coderouter_cli-2.13.0.dist-info → coderouter_cli-2.15.0.dist-info}/WHEEL +1 -1
- {coderouter_cli-2.13.0.dist-info → coderouter_cli-2.15.0.dist-info}/entry_points.txt +0 -0
- {coderouter_cli-2.13.0.dist-info → coderouter_cli-2.15.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -30,6 +30,7 @@ from typing import Any
|
|
|
30
30
|
from fastapi import APIRouter, Header, HTTPException, Request
|
|
31
31
|
from fastapi.responses import JSONResponse, StreamingResponse
|
|
32
32
|
|
|
33
|
+
from coderouter.adapters.base import StreamTruncatedError
|
|
33
34
|
from coderouter.guards.tool_loop import ToolCountExceededError, ToolLoopBreakError
|
|
34
35
|
from coderouter.logging import get_logger
|
|
35
36
|
from coderouter.routing import (
|
|
@@ -38,6 +39,10 @@ from coderouter.routing import (
|
|
|
38
39
|
NoProvidersAvailableError,
|
|
39
40
|
)
|
|
40
41
|
from coderouter.routing.auto_router import RESERVED_PROFILE_NAME, classify
|
|
42
|
+
from coderouter.routing.fallback_trace import (
|
|
43
|
+
SSE_FALLBACK_EVENT,
|
|
44
|
+
current_fallback_trace,
|
|
45
|
+
)
|
|
41
46
|
from coderouter.token_estimation import extract_text_from_anthropic_request
|
|
42
47
|
from coderouter.token_estimation_accurate import count_tokens, is_accuracy_available
|
|
43
48
|
from coderouter.translation import (
|
|
@@ -54,6 +59,11 @@ _ANTHROPIC_VERSION_HEADER = "anthropic-version"
|
|
|
54
59
|
_ANTHROPIC_BETA_HEADER = "anthropic-beta"
|
|
55
60
|
_CTX_BUDGET_HEADER = "X-CodeRouter-Context-Budget"
|
|
56
61
|
_DRIFT_HEADER = "X-CodeRouter-Drift"
|
|
62
|
+
# v2.15.0: the ``X-CodeRouter-Fallback-*`` family is built by
|
|
63
|
+
# ``FallbackTrace.header_values()`` rather than named here, because the set
|
|
64
|
+
# of headers varies with the trace (``-To`` is absent when the whole chain
|
|
65
|
+
# was exhausted). The names themselves live next to the trace so the
|
|
66
|
+
# engine, the ingress and the tests all read one definition.
|
|
57
67
|
|
|
58
68
|
# M14: overall SSE stream ceiling. A single streamed request can legitimately
|
|
59
69
|
# run much longer than one provider call (long generations, tool turns), so we
|
|
@@ -258,6 +268,20 @@ async def messages(
|
|
|
258
268
|
drift_severity = engine.last_drift_severity
|
|
259
269
|
if drift_severity:
|
|
260
270
|
stream_headers[_DRIFT_HEADER] = drift_severity
|
|
271
|
+
# v2.15.0: fallback headers on the streaming path carry what is
|
|
272
|
+
# already known when the response commits — i.e. the pre-attempt
|
|
273
|
+
# hops (paid gate / budget / backend health / self-healing) that
|
|
274
|
+
# ``apply_context_budget`` recorded while resolving the chain just
|
|
275
|
+
# above. Runtime attempt failures happen *after* the HTTP headers
|
|
276
|
+
# have shipped and are therefore physically unreachable here; they
|
|
277
|
+
# are delivered instead as the trailing ``coderouter_fallback`` SSE
|
|
278
|
+
# metadata event emitted by ``_anthropic_sse_iterator`` (the same
|
|
279
|
+
# trailer mechanism v2.0-H uses for ``coderouter_partial``), and as
|
|
280
|
+
# ``fallback-occurred`` log lines. Same commit-order constraint the
|
|
281
|
+
# drift header comment above describes.
|
|
282
|
+
pre_dispatch_trace = current_fallback_trace()
|
|
283
|
+
if pre_dispatch_trace is not None:
|
|
284
|
+
stream_headers.update(pre_dispatch_trace.header_values())
|
|
261
285
|
# M14: wrap the SSE iterator with an overall timeout + client
|
|
262
286
|
# disconnect cleanup so a wedged upstream cannot pin the client
|
|
263
287
|
# (or the upstream socket) open forever.
|
|
@@ -309,6 +333,15 @@ async def messages(
|
|
|
309
333
|
resp_headers[_CTX_BUDGET_HEADER] = ctx_budget_status
|
|
310
334
|
if drift_severity:
|
|
311
335
|
resp_headers[_DRIFT_HEADER] = drift_severity
|
|
336
|
+
# v2.15.0: fallback reason headers. The engine wrote its hops onto the
|
|
337
|
+
# request-scoped trace while ``generate_anthropic`` was awaited above,
|
|
338
|
+
# so by here the trail is complete — ``header_values()`` returns an
|
|
339
|
+
# empty dict (and this whole block is inert) when the first provider
|
|
340
|
+
# served the request, which keeps the healthy path byte-identical to
|
|
341
|
+
# v2.14.0.
|
|
342
|
+
fallback_trace = current_fallback_trace()
|
|
343
|
+
if fallback_trace is not None:
|
|
344
|
+
resp_headers.update(fallback_trace.header_values())
|
|
312
345
|
|
|
313
346
|
if resp_headers:
|
|
314
347
|
return JSONResponse(
|
|
@@ -511,6 +544,17 @@ async def _anthropic_sse_iterator(
|
|
|
511
544
|
except (KeyError, ValueError):
|
|
512
545
|
pass
|
|
513
546
|
|
|
547
|
+
# v2.15.0 (stream-truncation): name the specific failure on the
|
|
548
|
+
# metadata event. ``mid_stream_failure`` stays the default so every
|
|
549
|
+
# pre-v2.15.0 case is byte-identical; ``stream_truncated`` tells the
|
|
550
|
+
# client the difference between "the provider errored out" and "the
|
|
551
|
+
# provider went quiet with the message still open".
|
|
552
|
+
partial_reason = (
|
|
553
|
+
"stream_truncated"
|
|
554
|
+
if isinstance(exc.original, StreamTruncatedError)
|
|
555
|
+
else "mid_stream_failure"
|
|
556
|
+
)
|
|
557
|
+
|
|
514
558
|
if partial_action == "surface" and exc.partial_content:
|
|
515
559
|
# Emit message_delta with accumulated usage (signals stream end).
|
|
516
560
|
yield _format_anthropic_sse(AnthropicStreamEvent(
|
|
@@ -533,21 +577,32 @@ async def _anthropic_sse_iterator(
|
|
|
533
577
|
"type": "coderouter_partial",
|
|
534
578
|
"partial_content": exc.partial_content,
|
|
535
579
|
"provider": exc.provider,
|
|
536
|
-
"reason":
|
|
580
|
+
"reason": partial_reason,
|
|
537
581
|
"original_error": str(exc.original)[:200],
|
|
538
582
|
},
|
|
539
583
|
))
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
584
|
+
# v2.15.0: keep the pre-v2.15.0 key set (and order) intact for
|
|
585
|
+
# every failure that is not a truncation. The JSON formatter
|
|
586
|
+
# walks ``record.__dict__`` and emits whatever ``extra`` carries,
|
|
587
|
+
# so an unconditional ``reason`` would change the shape of this
|
|
588
|
+
# log line even under the default ``stream_truncation_action:
|
|
589
|
+
# off`` — where no truncation can be detected in the first place
|
|
590
|
+
# and the release claims byte-for-byte compatibility with
|
|
591
|
+
# v2.14.0. The key is therefore appended only when it says
|
|
592
|
+
# something new.
|
|
593
|
+
_stitch_extra: dict[str, Any] = {
|
|
594
|
+
"provider": exc.provider,
|
|
595
|
+
"profile": profile_name,
|
|
596
|
+
"text_blocks": len(exc.partial_content),
|
|
597
|
+
"text_length": sum(
|
|
598
|
+
len(b.get("text", "")) for b in exc.partial_content
|
|
599
|
+
),
|
|
600
|
+
}
|
|
601
|
+
if partial_reason != "mid_stream_failure":
|
|
602
|
+
# Lets a dashboard split "provider errored" from "provider
|
|
603
|
+
# went quiet" without re-parsing the SSE.
|
|
604
|
+
_stitch_extra["reason"] = partial_reason
|
|
605
|
+
logger.info("partial-stitch-surfaced", extra=_stitch_extra)
|
|
551
606
|
else:
|
|
552
607
|
err_event = AnthropicStreamEvent(
|
|
553
608
|
type="error",
|
|
@@ -561,6 +616,21 @@ async def _anthropic_sse_iterator(
|
|
|
561
616
|
)
|
|
562
617
|
yield _format_anthropic_sse(err_event)
|
|
563
618
|
|
|
619
|
+
# v2.15.0: trailing fallback metadata. Emitted after the stream has
|
|
620
|
+
# terminated (normally or with an in-stream error frame), so it never
|
|
621
|
+
# interleaves with the Anthropic event sequence a client is parsing —
|
|
622
|
+
# exactly the position ``coderouter_partial`` occupies. Client-optional:
|
|
623
|
+
# spec-compliant Anthropic SDKs ignore unknown event types, and the
|
|
624
|
+
# frame is only produced when a fallback actually happened.
|
|
625
|
+
trace = current_fallback_trace()
|
|
626
|
+
if trace is not None and trace.occurred:
|
|
627
|
+
yield _format_anthropic_sse(
|
|
628
|
+
AnthropicStreamEvent(
|
|
629
|
+
type=SSE_FALLBACK_EVENT,
|
|
630
|
+
data=trace.as_event_payload(),
|
|
631
|
+
)
|
|
632
|
+
)
|
|
633
|
+
|
|
564
634
|
|
|
565
635
|
def _format_anthropic_sse(ev: AnthropicStreamEvent) -> str:
|
|
566
636
|
"""Serialize an :class:`AnthropicStreamEvent` onto the SSE wire.
|
|
@@ -34,9 +34,11 @@ Why inline JS / CSS
|
|
|
34
34
|
|
|
35
35
|
from __future__ import annotations
|
|
36
36
|
|
|
37
|
-
from fastapi import APIRouter
|
|
37
|
+
from fastapi import APIRouter, Request
|
|
38
38
|
from fastapi.responses import HTMLResponse
|
|
39
39
|
|
|
40
|
+
from coderouter.ingress.ui_auth import auth_required, require_ui_token
|
|
41
|
+
|
|
40
42
|
router = APIRouter()
|
|
41
43
|
|
|
42
44
|
|
|
@@ -219,6 +221,17 @@ _DASHBOARD_HTML = r"""<!doctype html>
|
|
|
219
221
|
// Poll interval — 2s matches plan.md §12.3.5. Short enough to feel
|
|
220
222
|
// live on a local dev machine, long enough that HTTP overhead is noise.
|
|
221
223
|
const POLL_MS = 2000;
|
|
224
|
+
// v2.14.0: the token is entered by the operator and kept in
|
|
225
|
+
// sessionStorage (cleared when the tab closes). The page is told only
|
|
226
|
+
// whether auth is on, never what the token is.
|
|
227
|
+
const AUTH_REQUIRED = __METRICS_AUTH_REQUIRED__;
|
|
228
|
+
const TOKEN_KEY = "coderouter-metrics-token";
|
|
229
|
+
let metricsToken = sessionStorage.getItem(TOKEN_KEY) || "";
|
|
230
|
+
if (AUTH_REQUIRED && !metricsToken) {
|
|
231
|
+
const v = window.prompt("CODEROUTER_METRICS_TOKEN を入力してください", "");
|
|
232
|
+
if (v) { metricsToken = v.trim(); sessionStorage.setItem(TOKEN_KEY, metricsToken); }
|
|
233
|
+
}
|
|
234
|
+
const authHeaders = () => metricsToken ? {"X-CodeRouter-Token": metricsToken} : {};
|
|
222
235
|
// Sparkline ring — 60 samples at 2s/sample → 2 minutes of history.
|
|
223
236
|
const SPARK_POINTS = 60;
|
|
224
237
|
const sparkBuffer = [];
|
|
@@ -524,7 +537,7 @@ _DASHBOARD_HTML = r"""<!doctype html>
|
|
|
524
537
|
|
|
525
538
|
const poll = async () => {
|
|
526
539
|
try {
|
|
527
|
-
const resp = await fetch("/metrics.json", {cache: "no-store"});
|
|
540
|
+
const resp = await fetch("/metrics.json", {cache: "no-store", headers: authHeaders()});
|
|
528
541
|
if (!resp.ok) {
|
|
529
542
|
renderError("HTTP " + resp.status);
|
|
530
543
|
return;
|
|
@@ -547,7 +560,7 @@ _DASHBOARD_HTML = r"""<!doctype html>
|
|
|
547
560
|
|
|
548
561
|
|
|
549
562
|
@router.get("/dashboard", response_class=HTMLResponse)
|
|
550
|
-
async def dashboard() -> HTMLResponse:
|
|
563
|
+
async def dashboard(request: Request) -> HTMLResponse:
|
|
551
564
|
"""Serve the single-page dashboard HTML.
|
|
552
565
|
|
|
553
566
|
The page is entirely static — all data comes from polling
|
|
@@ -556,4 +569,11 @@ async def dashboard() -> HTMLResponse:
|
|
|
556
569
|
the template lives in this module as a string, keeping the shipping
|
|
557
570
|
unit a single Python file with no template dir to configure.
|
|
558
571
|
"""
|
|
559
|
-
|
|
572
|
+
require_ui_token(request)
|
|
573
|
+
# Only the boolean reaches the page — never the token. Same split the
|
|
574
|
+
# v2.13.0 launcher fix introduced after ``curl /launcher | grep``
|
|
575
|
+
# recovered the shared secret straight out of the HTML.
|
|
576
|
+
html = _DASHBOARD_HTML.replace(
|
|
577
|
+
"__METRICS_AUTH_REQUIRED__", "true" if auth_required() else "false"
|
|
578
|
+
)
|
|
579
|
+
return HTMLResponse(content=html)
|
|
@@ -28,7 +28,9 @@ from typing import Any
|
|
|
28
28
|
from fastapi import APIRouter, Request
|
|
29
29
|
from fastapi.responses import PlainTextResponse
|
|
30
30
|
|
|
31
|
+
from coderouter.ingress.ui_auth import require_ui_token
|
|
31
32
|
from coderouter.metrics import format_prometheus, get_collector
|
|
33
|
+
from coderouter.secret_redaction import redact
|
|
32
34
|
|
|
33
35
|
router = APIRouter()
|
|
34
36
|
|
|
@@ -48,6 +50,11 @@ async def metrics_json(request: Request) -> dict[str, Any]:
|
|
|
48
50
|
dashboard uses it to classify providers into local / free / paid
|
|
49
51
|
for the usage-mix bar without a second endpoint round-trip.
|
|
50
52
|
"""
|
|
53
|
+
# v2.14.0: opt-in token first, before anything is computed. The
|
|
54
|
+
# ordering is the point — codex-router's /health answers before its
|
|
55
|
+
# own auth check and leaks the live session name to any local caller.
|
|
56
|
+
require_ui_token(request)
|
|
57
|
+
|
|
51
58
|
snapshot = get_collector().snapshot()
|
|
52
59
|
|
|
53
60
|
config = getattr(request.app.state, "config", None)
|
|
@@ -66,7 +73,10 @@ async def metrics_json(request: Request) -> dict[str, Any]:
|
|
|
66
73
|
"paid": p.paid,
|
|
67
74
|
# ``HttpUrl`` is not JSON-serializable directly in Pydantic v2;
|
|
68
75
|
# the cast also makes the shape stable if Pydantic switches types.
|
|
69
|
-
|
|
76
|
+
# v2.14.0: run it through the scrubber. A base_url is the
|
|
77
|
+
# one config field an operator can paste a credential into
|
|
78
|
+
# (``?api_key=…``), and this endpoint hands it to a browser.
|
|
79
|
+
"base_url": redact(str(p.base_url)),
|
|
70
80
|
}
|
|
71
81
|
for p in config.providers
|
|
72
82
|
],
|
|
@@ -79,7 +89,7 @@ async def metrics_json(request: Request) -> dict[str, Any]:
|
|
|
79
89
|
|
|
80
90
|
|
|
81
91
|
@router.get("/metrics", response_class=PlainTextResponse)
|
|
82
|
-
async def metrics_prometheus() -> PlainTextResponse:
|
|
92
|
+
async def metrics_prometheus(request: Request) -> PlainTextResponse:
|
|
83
93
|
"""Prometheus text exposition format (v1.5-B).
|
|
84
94
|
|
|
85
95
|
Convention-compliant endpoint path for Prometheus scrapers. Returns
|
|
@@ -88,5 +98,6 @@ async def metrics_prometheus() -> PlainTextResponse:
|
|
|
88
98
|
Sits alongside :func:`metrics_json` (not a replacement) — JSON is
|
|
89
99
|
for internal UI, Prometheus is for external time-series DBs.
|
|
90
100
|
"""
|
|
101
|
+
require_ui_token(request)
|
|
91
102
|
body = format_prometheus(get_collector().snapshot())
|
|
92
103
|
return PlainTextResponse(content=body, media_type=_PROM_CONTENT_TYPE)
|
|
@@ -29,12 +29,13 @@ from collections.abc import AsyncIterator
|
|
|
29
29
|
from typing import Any
|
|
30
30
|
|
|
31
31
|
from fastapi import APIRouter, Header, HTTPException, Request
|
|
32
|
-
from fastapi.responses import StreamingResponse
|
|
32
|
+
from fastapi.responses import JSONResponse, StreamingResponse
|
|
33
33
|
|
|
34
34
|
from coderouter.adapters.base import ChatRequest
|
|
35
35
|
from coderouter.logging import get_logger
|
|
36
36
|
from coderouter.routing import FallbackEngine, NoProvidersAvailableError
|
|
37
37
|
from coderouter.routing.auto_router import RESERVED_PROFILE_NAME, classify
|
|
38
|
+
from coderouter.routing.fallback_trace import current_fallback_trace
|
|
38
39
|
|
|
39
40
|
router = APIRouter()
|
|
40
41
|
logger = get_logger(__name__)
|
|
@@ -179,7 +180,7 @@ async def chat_completions(
|
|
|
179
180
|
request: Request,
|
|
180
181
|
x_coderouter_profile: str | None = Header(default=None, alias=_PROFILE_HEADER),
|
|
181
182
|
x_coderouter_mode: str | None = Header(default=None, alias=_MODE_HEADER),
|
|
182
|
-
) -> StreamingResponse | dict[str, Any]:
|
|
183
|
+
) -> StreamingResponse | JSONResponse | dict[str, Any]:
|
|
183
184
|
"""OpenAI Chat Completions endpoint.
|
|
184
185
|
|
|
185
186
|
Validates the body into :class:`ChatRequest`, resolves the profile
|
|
@@ -252,6 +253,34 @@ async def chat_completions(
|
|
|
252
253
|
except NoProvidersAvailableError as exc:
|
|
253
254
|
raise HTTPException(status_code=502, detail=str(exc)) from exc
|
|
254
255
|
|
|
256
|
+
# v2.15.0: fallback reason headers, same shape and same construction as
|
|
257
|
+
# ``anthropic_routes.messages`` — the engine filled the request-scoped
|
|
258
|
+
# trace while ``generate`` was awaited, and ``header_values()`` is empty
|
|
259
|
+
# unless the chain actually moved. A request served by its first
|
|
260
|
+
# provider therefore still returns the bare dict, byte-identical to
|
|
261
|
+
# v2.14.0.
|
|
262
|
+
#
|
|
263
|
+
# Only the non-streaming half exists here, and that is deliberate. The
|
|
264
|
+
# Anthropic streaming path can carry the trail two ways: chain-resolve
|
|
265
|
+
# hops ride the HTTP headers (``apply_context_budget`` resolves the
|
|
266
|
+
# chain before the response commits) and runtime hops ride the trailing
|
|
267
|
+
# ``coderouter_fallback`` SSE event (Anthropic's wire is typed, so an
|
|
268
|
+
# unknown event type is ignored by spec-compliant SDKs). Neither is
|
|
269
|
+
# available on the OpenAI wire: this route has no pre-dispatch chain
|
|
270
|
+
# resolution, and OpenAI SSE frames are untyped — every ``data:`` line
|
|
271
|
+
# is parsed as a ChatCompletionChunk, so injecting a metadata frame
|
|
272
|
+
# would hand strict clients an object they cannot deserialize. That is
|
|
273
|
+
# a breaking change, so the streaming OpenAI path stays untouched and
|
|
274
|
+
# its reasons are available via the ``fallback-occurred`` log lines.
|
|
275
|
+
fallback_trace = current_fallback_trace()
|
|
276
|
+
if fallback_trace is not None:
|
|
277
|
+
resp_headers = fallback_trace.header_values()
|
|
278
|
+
if resp_headers:
|
|
279
|
+
return JSONResponse(
|
|
280
|
+
content=response.model_dump(exclude_none=True),
|
|
281
|
+
headers=resp_headers,
|
|
282
|
+
)
|
|
283
|
+
|
|
255
284
|
return response.model_dump(exclude_none=True)
|
|
256
285
|
|
|
257
286
|
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""Opt-in token auth for the read-only UI surfaces (v2.14.0).
|
|
2
|
+
|
|
3
|
+
``/dashboard``, ``/metrics.json`` and ``/metrics`` were the last endpoints
|
|
4
|
+
with no authentication story at all. They change nothing, so they were
|
|
5
|
+
never part of the H-8 state-changing-endpoint work — but "read-only" is
|
|
6
|
+
not "harmless". ``/metrics.json`` returns every provider's name, kind,
|
|
7
|
+
paid flag and ``base_url``, plus the profile graph: the full topology of
|
|
8
|
+
which models an operator runs and which vendors they pay. On a laptop
|
|
9
|
+
that is fine. On a box where the port is reachable by anything else, it
|
|
10
|
+
is a free reconnaissance endpoint.
|
|
11
|
+
|
|
12
|
+
The failure we are deliberately not repeating is codex-router's: its
|
|
13
|
+
``/health`` answers *before* the caller-key check and includes the live
|
|
14
|
+
session name, so any local process can read what the operator is working
|
|
15
|
+
on. Cheap to avoid, easy to ship by accident — the check just has to come
|
|
16
|
+
first.
|
|
17
|
+
|
|
18
|
+
Contract
|
|
19
|
+
--------
|
|
20
|
+
Identical in shape to :func:`coderouter.ingress.launcher_routes._require_launcher_token`,
|
|
21
|
+
so an operator learns it once:
|
|
22
|
+
|
|
23
|
+
* ``CODEROUTER_METRICS_TOKEN`` unset → the endpoints stay open exactly as
|
|
24
|
+
they were, and a one-time warning is logged. Backwards compatible by
|
|
25
|
+
construction; nobody's Prometheus scrape breaks on upgrade.
|
|
26
|
+
* Set → every request must carry a matching ``X-CodeRouter-Token`` header,
|
|
27
|
+
compared with :func:`secrets.compare_digest`.
|
|
28
|
+
|
|
29
|
+
The token is deliberately **not** accepted as a query parameter. A token
|
|
30
|
+
in a URL lands in access logs, in ``Referer`` headers, and in the browser
|
|
31
|
+
history — and this codebase now has a log scrubber precisely because
|
|
32
|
+
credentials end up in places nobody predicted.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
from __future__ import annotations
|
|
36
|
+
|
|
37
|
+
import os
|
|
38
|
+
import secrets
|
|
39
|
+
|
|
40
|
+
from fastapi import HTTPException, Request
|
|
41
|
+
|
|
42
|
+
from coderouter.logging import get_logger
|
|
43
|
+
|
|
44
|
+
__all__ = ["METRICS_TOKEN_ENV", "METRICS_TOKEN_HEADER", "require_ui_token"]
|
|
45
|
+
|
|
46
|
+
logger = get_logger(__name__)
|
|
47
|
+
|
|
48
|
+
METRICS_TOKEN_ENV = "CODEROUTER_METRICS_TOKEN"
|
|
49
|
+
METRICS_TOKEN_HEADER = "X-CodeRouter-Token"
|
|
50
|
+
|
|
51
|
+
# One warning per process, not per request — a 2-second dashboard poll
|
|
52
|
+
# would otherwise bury every other log line.
|
|
53
|
+
_warning_emitted = False
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def require_ui_token(request: Request) -> None:
|
|
57
|
+
"""Enforce the metrics/dashboard token when one is configured.
|
|
58
|
+
|
|
59
|
+
Raises ``401`` on mismatch. Returns silently when the env var is
|
|
60
|
+
unset, after logging a single ``metrics-auth-disabled`` line.
|
|
61
|
+
"""
|
|
62
|
+
global _warning_emitted
|
|
63
|
+
expected = os.environ.get(METRICS_TOKEN_ENV, "")
|
|
64
|
+
if not expected:
|
|
65
|
+
if not _warning_emitted:
|
|
66
|
+
logger.warning(
|
|
67
|
+
"metrics-auth-disabled",
|
|
68
|
+
extra={
|
|
69
|
+
"hint": (
|
|
70
|
+
f"{METRICS_TOKEN_ENV} is not set; /dashboard, "
|
|
71
|
+
"/metrics.json and /metrics are unauthenticated and "
|
|
72
|
+
"expose provider names, base_urls and the profile "
|
|
73
|
+
"graph. Set it when binding to anything but loopback."
|
|
74
|
+
),
|
|
75
|
+
},
|
|
76
|
+
)
|
|
77
|
+
_warning_emitted = True
|
|
78
|
+
return
|
|
79
|
+
provided = request.headers.get(METRICS_TOKEN_HEADER, "")
|
|
80
|
+
if not secrets.compare_digest(provided, expected):
|
|
81
|
+
raise HTTPException(
|
|
82
|
+
status_code=401, detail="Invalid or missing metrics token."
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def auth_required() -> bool:
|
|
87
|
+
"""Whether a token is configured — safe to embed in the page.
|
|
88
|
+
|
|
89
|
+
The dashboard needs to know *that* auth is on so it can prompt; it
|
|
90
|
+
must never receive the token itself. Same split as the v2.13.0 fix
|
|
91
|
+
that stopped ``GET /launcher`` embedding ``CODEROUTER_LAUNCHER_TOKEN``
|
|
92
|
+
straight into the HTML, where ``curl | grep`` recovered it.
|
|
93
|
+
"""
|
|
94
|
+
return bool(os.environ.get(METRICS_TOKEN_ENV, ""))
|
coderouter/logging.py
CHANGED
|
@@ -23,6 +23,8 @@ import logging
|
|
|
23
23
|
import sys
|
|
24
24
|
from typing import Any, Literal, TypedDict
|
|
25
25
|
|
|
26
|
+
from coderouter.secret_redaction import install_secret_filter
|
|
27
|
+
|
|
26
28
|
|
|
27
29
|
class JsonLineFormatter(logging.Formatter):
|
|
28
30
|
"""Emit each record as a single JSON line."""
|
|
@@ -105,6 +107,11 @@ def configure_logging(level: str = "INFO") -> None:
|
|
|
105
107
|
root.removeHandler(h)
|
|
106
108
|
handler = logging.StreamHandler(sys.stderr)
|
|
107
109
|
handler.setFormatter(JsonLineFormatter())
|
|
110
|
+
# v2.14.0: scrub registered credentials before the formatter runs. The
|
|
111
|
+
# filter is attached per-handler (not to the root logger) because logger
|
|
112
|
+
# filters do not apply to records propagated up from child loggers —
|
|
113
|
+
# every emitting module in this codebase uses its own module logger.
|
|
114
|
+
install_secret_filter(handler)
|
|
108
115
|
setattr(handler, _CODEROUTER_LOG_HANDLER_MARKER, True)
|
|
109
116
|
root.addHandler(handler)
|
|
110
117
|
|
|
@@ -1472,6 +1479,63 @@ def log_empty_response_detected(
|
|
|
1472
1479
|
)
|
|
1473
1480
|
|
|
1474
1481
|
|
|
1482
|
+
# ---------------------------------------------------------------------------
|
|
1483
|
+
# v2.15.0 (stream-truncation): upstream SSE ended without a terminator
|
|
1484
|
+
#
|
|
1485
|
+
# Fired from the adapter's SSE parse loop when ``stream_truncation_action``
|
|
1486
|
+
# is ``warn`` or ``error`` and the upstream line iterator was exhausted
|
|
1487
|
+
# without a ``message_stop`` (Anthropic wire) / ``[DONE]`` or a
|
|
1488
|
+
# ``finish_reason`` (OpenAI wire).
|
|
1489
|
+
#
|
|
1490
|
+
# ``action`` records which knob value produced the line, so a dashboard can
|
|
1491
|
+
# separate "we only measured it" (``warn``) from "we acted on it" (``error``).
|
|
1492
|
+
# ``events_forwarded`` is how many events/chunks the adapter had already
|
|
1493
|
+
# yielded — 0 means nothing reached the engine and a clean fallback is still
|
|
1494
|
+
# possible. ``tool_call_in_flight`` is the sharp edge: True means a
|
|
1495
|
+
# ``tool_use`` / ``tool_calls`` block was still open, so whatever the
|
|
1496
|
+
# translation layer synthesizes closes a block whose argument JSON is
|
|
1497
|
+
# provably incomplete.
|
|
1498
|
+
# ---------------------------------------------------------------------------
|
|
1499
|
+
|
|
1500
|
+
|
|
1501
|
+
def log_stream_truncation_detected(
|
|
1502
|
+
logger: logging.Logger,
|
|
1503
|
+
provider: str,
|
|
1504
|
+
*,
|
|
1505
|
+
action: str,
|
|
1506
|
+
wire: str,
|
|
1507
|
+
events_forwarded: int,
|
|
1508
|
+
saw_stream_start: bool,
|
|
1509
|
+
tool_call_in_flight: bool = False,
|
|
1510
|
+
) -> None:
|
|
1511
|
+
"""Emit a ``stream-truncation-detected`` warn line.
|
|
1512
|
+
|
|
1513
|
+
Args:
|
|
1514
|
+
logger: Target logger.
|
|
1515
|
+
provider: ``ProviderConfig.name`` whose stream was cut.
|
|
1516
|
+
action: The ``stream_truncation_action`` value in effect (``warn``
|
|
1517
|
+
or ``error`` — ``off`` never reaches here).
|
|
1518
|
+
wire: ``anthropic`` or ``openai`` — which SSE dialect was being
|
|
1519
|
+
parsed, i.e. which terminator went missing.
|
|
1520
|
+
events_forwarded: Count of events/chunks already yielded downstream.
|
|
1521
|
+
saw_stream_start: Whether the opening event (``message_start`` /
|
|
1522
|
+
first data chunk) had arrived at all. False means the upstream
|
|
1523
|
+
produced nothing before closing.
|
|
1524
|
+
tool_call_in_flight: Whether a tool-call block was still open.
|
|
1525
|
+
"""
|
|
1526
|
+
logger.warning(
|
|
1527
|
+
"stream-truncation-detected",
|
|
1528
|
+
extra={
|
|
1529
|
+
"provider": provider,
|
|
1530
|
+
"action": action,
|
|
1531
|
+
"wire": wire,
|
|
1532
|
+
"events_forwarded": events_forwarded,
|
|
1533
|
+
"saw_stream_start": saw_stream_start,
|
|
1534
|
+
"tool_call_in_flight": tool_call_in_flight,
|
|
1535
|
+
},
|
|
1536
|
+
)
|
|
1537
|
+
|
|
1538
|
+
|
|
1475
1539
|
# ---------------------------------------------------------------------------
|
|
1476
1540
|
# v2.0-I: Continuous probe log shapes
|
|
1477
1541
|
#
|
|
@@ -1605,3 +1669,89 @@ def log_probe_capabilities_drift(
|
|
|
1605
1669
|
"in_registry": in_registry,
|
|
1606
1670
|
}
|
|
1607
1671
|
logger.warning("probe-capabilities-drift", extra=payload)
|
|
1672
|
+
|
|
1673
|
+
|
|
1674
|
+
# ---------------------------------------------------------------------------
|
|
1675
|
+
# v2.15.0: fallback reason logging
|
|
1676
|
+
#
|
|
1677
|
+
# One ``fallback-occurred`` line per hop. Fires only when the chain
|
|
1678
|
+
# actually moves — a request served by its first provider produces no
|
|
1679
|
+
# line at all, so this adds zero volume to the healthy path.
|
|
1680
|
+
#
|
|
1681
|
+
# The vocabulary of ``reason`` is defined in
|
|
1682
|
+
# :mod:`coderouter.routing.fallback_trace` (a single canonical set that
|
|
1683
|
+
# also names the pre-existing ``provider-does-not-support`` /
|
|
1684
|
+
# ``translation-lossy`` degrade reasons, which never appear here — a
|
|
1685
|
+
# degrade keeps the request on the same provider).
|
|
1686
|
+
#
|
|
1687
|
+
# ``to_provider`` is ``None`` when the hop is the chain's last one and no
|
|
1688
|
+
# successor was ever attempted (chain exhausted). The line is still
|
|
1689
|
+
# emitted: "we left A because of R and had nowhere to go" is the single
|
|
1690
|
+
# most diagnostic event in the whole engine.
|
|
1691
|
+
# ---------------------------------------------------------------------------
|
|
1692
|
+
|
|
1693
|
+
|
|
1694
|
+
class FallbackOccurredPayload(TypedDict):
|
|
1695
|
+
"""Structured shape of the ``fallback-occurred`` log record.
|
|
1696
|
+
|
|
1697
|
+
Fields
|
|
1698
|
+
provider: the provider being left. Duplicated from
|
|
1699
|
+
``from_provider`` under the field name every other engine
|
|
1700
|
+
event uses, so existing per-provider log queries keep working
|
|
1701
|
+
without learning a new key.
|
|
1702
|
+
from_provider: the provider being left.
|
|
1703
|
+
to_provider: the provider taking over, or None if the chain was
|
|
1704
|
+
exhausted with no successor.
|
|
1705
|
+
reason: canonical fallback reason (see ``fallback_trace``).
|
|
1706
|
+
detail: short human-readable context (HTTP status, truncated
|
|
1707
|
+
error text), or None.
|
|
1708
|
+
profile: the resolved profile name, when known.
|
|
1709
|
+
stream: True when the hop happened on a streaming request.
|
|
1710
|
+
pre_attempt: True when the provider was filtered out during chain
|
|
1711
|
+
resolution (budget / health / paid gate) and therefore never
|
|
1712
|
+
actually called; False when it was called and failed.
|
|
1713
|
+
hop_index: 0-based position of this hop within the request.
|
|
1714
|
+
"""
|
|
1715
|
+
|
|
1716
|
+
provider: str
|
|
1717
|
+
from_provider: str
|
|
1718
|
+
to_provider: str | None
|
|
1719
|
+
reason: str
|
|
1720
|
+
detail: str | None
|
|
1721
|
+
profile: str | None
|
|
1722
|
+
stream: bool
|
|
1723
|
+
pre_attempt: bool
|
|
1724
|
+
hop_index: int
|
|
1725
|
+
|
|
1726
|
+
|
|
1727
|
+
def log_fallback_occurred(
|
|
1728
|
+
logger: logging.Logger,
|
|
1729
|
+
*,
|
|
1730
|
+
from_provider: str,
|
|
1731
|
+
to_provider: str | None,
|
|
1732
|
+
reason: str,
|
|
1733
|
+
detail: str | None = None,
|
|
1734
|
+
profile: str | None = None,
|
|
1735
|
+
stream: bool = False,
|
|
1736
|
+
pre_attempt: bool = False,
|
|
1737
|
+
hop_index: int = 0,
|
|
1738
|
+
) -> None:
|
|
1739
|
+
"""Emit a ``fallback-occurred`` warning line for one chain transition.
|
|
1740
|
+
|
|
1741
|
+
Warn level, matching ``provider-failed`` — a fallback is a
|
|
1742
|
+
degradation of the intended routing even when the request ultimately
|
|
1743
|
+
succeeds, and operators grep for it when a profile's primary provider
|
|
1744
|
+
stops earning its place at the front of the chain.
|
|
1745
|
+
"""
|
|
1746
|
+
payload: FallbackOccurredPayload = {
|
|
1747
|
+
"provider": from_provider,
|
|
1748
|
+
"from_provider": from_provider,
|
|
1749
|
+
"to_provider": to_provider,
|
|
1750
|
+
"reason": reason,
|
|
1751
|
+
"detail": detail,
|
|
1752
|
+
"profile": profile,
|
|
1753
|
+
"stream": stream,
|
|
1754
|
+
"pre_attempt": pre_attempt,
|
|
1755
|
+
"hop_index": hop_index,
|
|
1756
|
+
}
|
|
1757
|
+
logger.warning("fallback-occurred", extra=payload)
|
coderouter/metrics/collector.py
CHANGED
|
@@ -53,6 +53,8 @@ Event inventory (dispatch table in :meth:`MetricsCollector._dispatch`)
|
|
|
53
53
|
``drift-reload-attempted`` → ``drift_reload_total`` / success
|
|
54
54
|
``partial-stitch-surfaced`` → ``partial_stitch_surfaced_total`` (v2.0-H)
|
|
55
55
|
``empty-response-detected`` → ``empty_responses_total`` + per-provider (⑧)
|
|
56
|
+
``stream-truncation-detected`` → ``stream_truncated_total`` + per-provider
|
|
57
|
+
+ per-action (v2.15.0)
|
|
56
58
|
``probe-completed`` (v2.0-I) → ``probe_total`` / ``probe_success`` / ``probe_failure``
|
|
57
59
|
+ per-provider latency gauge
|
|
58
60
|
``probe-round-completed`` → ``probe_rounds_total`` (v2.0-I)
|
|
@@ -111,6 +113,7 @@ _KNOWN_EVENTS: Final[frozenset[str]] = frozenset(
|
|
|
111
113
|
"drift-reload-attempted",
|
|
112
114
|
"partial-stitch-surfaced",
|
|
113
115
|
"empty-response-detected",
|
|
116
|
+
"stream-truncation-detected",
|
|
114
117
|
"probe-completed",
|
|
115
118
|
"probe-round-completed",
|
|
116
119
|
"probe-capabilities-drift",
|
|
@@ -281,6 +284,16 @@ class MetricsCollector(logging.Handler):
|
|
|
281
284
|
self._empty_responses_total: int = 0
|
|
282
285
|
self._empty_responses_by_provider: Counter[str] = Counter()
|
|
283
286
|
|
|
287
|
+
# v2.15.0 (stream-truncation): count of upstream SSE streams that
|
|
288
|
+
# ended without a protocol terminator, under either ``warn`` or
|
|
289
|
+
# ``error``. Per-provider answers "which backend goes quiet"; the
|
|
290
|
+
# per-action split separates measurement from enforcement so a
|
|
291
|
+
# dashboard can show the ``warn``-phase rate before an operator
|
|
292
|
+
# commits to ``error``.
|
|
293
|
+
self._stream_truncated_total: int = 0
|
|
294
|
+
self._stream_truncated_by_provider: Counter[str] = Counter()
|
|
295
|
+
self._stream_truncated_by_action: Counter[str] = Counter()
|
|
296
|
+
|
|
284
297
|
# v2.0-I: continuous probe counters. Per-provider probe attempts
|
|
285
298
|
# and outcomes, plus a round counter for the dashboard's
|
|
286
299
|
# "probes/min" panel.
|
|
@@ -339,6 +352,7 @@ class MetricsCollector(logging.Handler):
|
|
|
339
352
|
"drift-reload-attempted": self._on_drift_reload_attempted,
|
|
340
353
|
"partial-stitch-surfaced": self._on_partial_stitch_surfaced,
|
|
341
354
|
"empty-response-detected": self._on_empty_response_detected,
|
|
355
|
+
"stream-truncation-detected": self._on_stream_truncation_detected,
|
|
342
356
|
"probe-completed": self._on_probe_completed,
|
|
343
357
|
"probe-round-completed": self._on_probe_round_completed,
|
|
344
358
|
"probe-capabilities-drift": self._on_probe_capabilities_drift,
|
|
@@ -691,6 +705,20 @@ class MetricsCollector(logging.Handler):
|
|
|
691
705
|
self._empty_responses_by_provider[provider] += 1
|
|
692
706
|
self._push_recent("empty-response-detected", extras, record)
|
|
693
707
|
|
|
708
|
+
def _on_stream_truncation_detected(
|
|
709
|
+
self, extras: dict[str, Any], record: logging.LogRecord
|
|
710
|
+
) -> None:
|
|
711
|
+
# v2.15.0 (stream-truncation): an upstream SSE stream ended without
|
|
712
|
+
# its terminator under a ``warn`` / ``error`` action.
|
|
713
|
+
self._stream_truncated_total += 1
|
|
714
|
+
provider = _str(extras.get("provider"))
|
|
715
|
+
if provider:
|
|
716
|
+
self._stream_truncated_by_provider[provider] += 1
|
|
717
|
+
action = _str(extras.get("action"))
|
|
718
|
+
if action:
|
|
719
|
+
self._stream_truncated_by_action[action] += 1
|
|
720
|
+
self._push_recent("stream-truncation-detected", extras, record)
|
|
721
|
+
|
|
694
722
|
def _on_probe_completed(
|
|
695
723
|
self, extras: dict[str, Any], record: logging.LogRecord
|
|
696
724
|
) -> None:
|
|
@@ -914,6 +942,14 @@ class MetricsCollector(logging.Handler):
|
|
|
914
942
|
"empty_responses_by_provider": dict(
|
|
915
943
|
self._empty_responses_by_provider
|
|
916
944
|
),
|
|
945
|
+
# v2.15.0 (stream-truncation).
|
|
946
|
+
"stream_truncated_total": self._stream_truncated_total,
|
|
947
|
+
"stream_truncated_by_provider": dict(
|
|
948
|
+
self._stream_truncated_by_provider
|
|
949
|
+
),
|
|
950
|
+
"stream_truncated_by_action": dict(
|
|
951
|
+
self._stream_truncated_by_action
|
|
952
|
+
),
|
|
917
953
|
# v2.0-I: continuous probe counters.
|
|
918
954
|
"probe_total": dict(self._probe_total),
|
|
919
955
|
"probe_success": dict(self._probe_success),
|
|
@@ -1071,6 +1107,10 @@ class MetricsCollector(logging.Handler):
|
|
|
1071
1107
|
# ⑧ (empty-response)
|
|
1072
1108
|
self._empty_responses_total = 0
|
|
1073
1109
|
self._empty_responses_by_provider.clear()
|
|
1110
|
+
# v2.15.0 (stream-truncation)
|
|
1111
|
+
self._stream_truncated_total = 0
|
|
1112
|
+
self._stream_truncated_by_provider.clear()
|
|
1113
|
+
self._stream_truncated_by_action.clear()
|
|
1074
1114
|
# v2.0-I
|
|
1075
1115
|
self._probe_total.clear()
|
|
1076
1116
|
self._probe_success.clear()
|