coderouter-cli 2.6.1__py3-none-any.whl → 2.7.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- coderouter/adapters/anthropic_native.py +66 -30
- coderouter/adapters/base.py +46 -3
- coderouter/adapters/openai_compat.py +53 -11
- coderouter/config/capability_registry.py +22 -0
- coderouter/config/schemas.py +232 -0
- coderouter/guards/context_budget.py +114 -19
- coderouter/guards/continuous_probe.py +9 -1
- coderouter/guards/memory_pressure.py +16 -2
- coderouter/guards/self_healing.py +14 -0
- coderouter/ingress/anthropic_routes.py +223 -1
- coderouter/ingress/app.py +162 -1
- coderouter/ingress/launcher_routes.py +187 -11
- coderouter/ingress/openai_routes.py +77 -7
- coderouter/logging.py +29 -3
- coderouter/metrics/collector.py +454 -245
- coderouter/metrics/prometheus.py +3 -3
- coderouter/routing/capability.py +217 -0
- coderouter/routing/fallback.py +539 -27
- coderouter/state/audit_log.py +69 -7
- coderouter/state/request_log.py +70 -7
- coderouter/translation/convert.py +112 -9
- coderouter/translation/tool_repair.py +671 -50
- {coderouter_cli-2.6.1.dist-info → coderouter_cli-2.7.1.dist-info}/METADATA +28 -9
- {coderouter_cli-2.6.1.dist-info → coderouter_cli-2.7.1.dist-info}/RECORD +27 -27
- {coderouter_cli-2.6.1.dist-info → coderouter_cli-2.7.1.dist-info}/WHEEL +0 -0
- {coderouter_cli-2.6.1.dist-info → coderouter_cli-2.7.1.dist-info}/entry_points.txt +0 -0
- {coderouter_cli-2.6.1.dist-info → coderouter_cli-2.7.1.dist-info}/licenses/LICENSE +0 -0
|
@@ -339,33 +339,128 @@ def _compute_preserve_set(
|
|
|
339
339
|
return preserved
|
|
340
340
|
|
|
341
341
|
|
|
342
|
+
def _group_removable_units(
|
|
343
|
+
messages: list[Any],
|
|
344
|
+
removable: list[int],
|
|
345
|
+
) -> list[list[int]]:
|
|
346
|
+
"""Group removable message indices into atomic tool-pair units.
|
|
347
|
+
|
|
348
|
+
``removable`` is the sorted list of indices eligible for removal
|
|
349
|
+
(i.e. not in the preserve set). A tool_use assistant message and the
|
|
350
|
+
user message carrying its matching tool_result must be removed
|
|
351
|
+
together so we never leave an orphaned tool_use or tool_result in the
|
|
352
|
+
trimmed history (Anthropic rejects both with a 400). Because the
|
|
353
|
+
preserve-set computation already keeps pairs atomic, both halves of a
|
|
354
|
+
pair are always either both preserved or both removable — here we just
|
|
355
|
+
coalesce the removable halves into a single unit so the incremental
|
|
356
|
+
loop drops them in one step.
|
|
357
|
+
|
|
358
|
+
Returns a list of units (each a sorted list of indices), ordered by
|
|
359
|
+
the position of the unit's earliest message so the caller can peel
|
|
360
|
+
them off the front oldest-first.
|
|
361
|
+
"""
|
|
362
|
+
removable_set = set(removable)
|
|
363
|
+
|
|
364
|
+
# Map every tool_use_id to the indices of the message(s) that emit the
|
|
365
|
+
# tool_use and the message that carries the matching tool_result.
|
|
366
|
+
partner: dict[int, set[int]] = {idx: set() for idx in removable}
|
|
367
|
+
use_index: dict[str, int] = {}
|
|
368
|
+
result_index: dict[str, int] = {}
|
|
369
|
+
for idx in removable:
|
|
370
|
+
for tid in _extract_tool_use_ids(messages[idx]):
|
|
371
|
+
use_index[tid] = idx
|
|
372
|
+
for tid in _extract_tool_result_ids(messages[idx]):
|
|
373
|
+
result_index[tid] = idx
|
|
374
|
+
for tid, use_idx in use_index.items():
|
|
375
|
+
res_idx = result_index.get(tid)
|
|
376
|
+
if res_idx is not None and res_idx in removable_set:
|
|
377
|
+
partner[use_idx].add(res_idx)
|
|
378
|
+
partner[res_idx].add(use_idx)
|
|
379
|
+
|
|
380
|
+
# Union-find style grouping over the partner graph.
|
|
381
|
+
seen: set[int] = set()
|
|
382
|
+
units: list[list[int]] = []
|
|
383
|
+
for idx in removable:
|
|
384
|
+
if idx in seen:
|
|
385
|
+
continue
|
|
386
|
+
stack = [idx]
|
|
387
|
+
group: set[int] = set()
|
|
388
|
+
while stack:
|
|
389
|
+
cur = stack.pop()
|
|
390
|
+
if cur in group:
|
|
391
|
+
continue
|
|
392
|
+
group.add(cur)
|
|
393
|
+
seen.add(cur)
|
|
394
|
+
stack.extend(p for p in partner[cur] if p not in group)
|
|
395
|
+
units.append(sorted(group))
|
|
396
|
+
|
|
397
|
+
units.sort(key=lambda unit: unit[0])
|
|
398
|
+
return units
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def _normalize_head(messages: list[Any]) -> list[Any]:
|
|
402
|
+
"""Drop leading messages until the first is a clean ``user`` message.
|
|
403
|
+
|
|
404
|
+
Anthropic requires the first message to be a ``user`` message, and a
|
|
405
|
+
leading user message must not open with a dangling ``tool_result``
|
|
406
|
+
(its ``tool_use`` was trimmed away). We therefore drop from the front
|
|
407
|
+
any assistant message and any user message that contains a
|
|
408
|
+
tool_result, stopping at the first ``user`` message with no
|
|
409
|
+
tool_result block.
|
|
410
|
+
"""
|
|
411
|
+
start = 0
|
|
412
|
+
n = len(messages)
|
|
413
|
+
while start < n:
|
|
414
|
+
msg = messages[start]
|
|
415
|
+
role = msg.role if hasattr(msg, "role") else (
|
|
416
|
+
msg.get("role") if isinstance(msg, dict) else None
|
|
417
|
+
)
|
|
418
|
+
if role == "user" and not _has_tool_result(msg):
|
|
419
|
+
break
|
|
420
|
+
start += 1
|
|
421
|
+
return messages[start:]
|
|
422
|
+
|
|
423
|
+
|
|
342
424
|
def _do_trim(
|
|
343
425
|
messages: list[Any],
|
|
344
426
|
system: Any,
|
|
345
427
|
target_tokens: int,
|
|
346
428
|
preserve_last_n: int,
|
|
347
429
|
) -> list[Any]:
|
|
348
|
-
"""
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
430
|
+
"""Incrementally drop oldest messages until under ``target_tokens``.
|
|
431
|
+
|
|
432
|
+
Unlike the previous implementation (which deleted every non-preserved
|
|
433
|
+
message in one shot the moment the threshold was crossed), this peels
|
|
434
|
+
messages off the front one atomic unit at a time and re-estimates
|
|
435
|
+
after each removal, stopping as soon as the estimate is at or below
|
|
436
|
+
the target. The preserve set (last N messages + their tool pairs) is a
|
|
437
|
+
hard floor that is never removed. After trimming, the head is
|
|
438
|
+
normalized so the first surviving message is a ``user`` message
|
|
439
|
+
without a leading ``tool_result`` (avoids upstream 400s). Bug H5.
|
|
440
|
+
"""
|
|
441
|
+
if not messages:
|
|
442
|
+
return messages
|
|
443
|
+
|
|
444
|
+
already = estimate_tokens_from_anthropic_request(system=system, messages=messages)
|
|
445
|
+
if already <= target_tokens:
|
|
446
|
+
# Nothing to do — return the input unchanged.
|
|
447
|
+
return messages
|
|
448
|
+
|
|
449
|
+
preserved_indices = _compute_preserve_set(messages, preserve_last_n)
|
|
450
|
+
removable = [i for i in range(len(messages)) if i not in preserved_indices]
|
|
451
|
+
units = _group_removable_units(messages, removable)
|
|
452
|
+
|
|
453
|
+
# Indices we have decided to drop; peel atomic units from the front.
|
|
454
|
+
dropped: set[int] = set()
|
|
455
|
+
for unit in units:
|
|
456
|
+
kept = [messages[i] for i in range(len(messages)) if i not in dropped]
|
|
457
|
+
estimated = estimate_tokens_from_anthropic_request(system=system, messages=kept)
|
|
360
458
|
if estimated <= target_tokens:
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
# Still over target — reduce preserve count and retry
|
|
364
|
-
current_preserve -= 1
|
|
459
|
+
break
|
|
460
|
+
dropped.update(unit)
|
|
365
461
|
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
return [messages[i] for i in sorted(preserved_indices)]
|
|
462
|
+
trimmed = [messages[i] for i in range(len(messages)) if i not in dropped]
|
|
463
|
+
return _normalize_head(trimmed)
|
|
369
464
|
|
|
370
465
|
|
|
371
466
|
__all__ = [
|
|
@@ -98,6 +98,11 @@ async def probe_one(
|
|
|
98
98
|
"""
|
|
99
99
|
import os
|
|
100
100
|
|
|
101
|
+
# Local import: keeps the module-level import graph free of the
|
|
102
|
+
# convert ↔ anthropic_native circular dependency (adapters pull in
|
|
103
|
+
# translation.convert, which pulls back in this adapter).
|
|
104
|
+
from coderouter.adapters.anthropic_native import anthropic_messages_url
|
|
105
|
+
|
|
101
106
|
start = time.monotonic()
|
|
102
107
|
provider_name = provider.name
|
|
103
108
|
base_url = str(provider.base_url).rstrip("/")
|
|
@@ -116,7 +121,10 @@ async def probe_one(
|
|
|
116
121
|
try:
|
|
117
122
|
async with httpx.AsyncClient(timeout=timeout_s) as client:
|
|
118
123
|
if provider.kind == "anthropic":
|
|
119
|
-
|
|
124
|
+
# Use the shared normalizer so the probe hits the exact
|
|
125
|
+
# same endpoint the adapter does — base_url ending in
|
|
126
|
+
# `/v1` must not produce `/v1/v1/messages` (bug H4).
|
|
127
|
+
url = anthropic_messages_url(base_url)
|
|
120
128
|
body: dict[str, Any] = {
|
|
121
129
|
"model": provider.model,
|
|
122
130
|
"max_tokens": 1,
|
|
@@ -53,6 +53,7 @@ the body of its read/write.
|
|
|
53
53
|
|
|
54
54
|
from __future__ import annotations
|
|
55
55
|
|
|
56
|
+
import re
|
|
56
57
|
import threading
|
|
57
58
|
import time
|
|
58
59
|
|
|
@@ -76,9 +77,18 @@ _MEMORY_PRESSURE_PHRASES: tuple[str, ...] = (
|
|
|
76
77
|
"failed to allocate",
|
|
77
78
|
"failed to load model",
|
|
78
79
|
"ggml_cuda_host_malloc", # llama.cpp specific
|
|
79
|
-
"oom",
|
|
80
80
|
)
|
|
81
81
|
|
|
82
|
+
# M10: "oom" needs word-boundary matching, not a bare substring test.
|
|
83
|
+
# A plain ``"oom" in text`` false-fires on "room" / "zoom" / "bloom" and
|
|
84
|
+
# would wrongly cool down a healthy provider whose error body merely
|
|
85
|
+
# contains one of those words. ``\boom\b`` matches the standalone token
|
|
86
|
+
# "oom" (and "OOM" once lowercased) while rejecting the embedded cases.
|
|
87
|
+
# Note the digit/underscore edges are word chars, so "oom_killer" is NOT
|
|
88
|
+
# matched by this pattern alone — real oom-killer bodies still trip the
|
|
89
|
+
# match via the separate "out of memory" phrase or a standalone "oom".
|
|
90
|
+
_OOM_TOKEN_RE = re.compile(r"\boom\b")
|
|
91
|
+
|
|
82
92
|
|
|
83
93
|
def is_memory_pressure_error(exc: AdapterError) -> bool:
|
|
84
94
|
"""Return True iff ``exc`` looks like a backend OOM signal.
|
|
@@ -96,7 +106,11 @@ def is_memory_pressure_error(exc: AdapterError) -> bool:
|
|
|
96
106
|
focused on the actual signal.
|
|
97
107
|
"""
|
|
98
108
|
text = str(exc).lower()
|
|
99
|
-
|
|
109
|
+
if any(phrase in text for phrase in _MEMORY_PRESSURE_PHRASES):
|
|
110
|
+
return True
|
|
111
|
+
# M10: standalone "oom" token only (word-boundary), so "room" /
|
|
112
|
+
# "zoom" / "bloom" in an unrelated error body don't false-fire.
|
|
113
|
+
return _OOM_TOKEN_RE.search(text) is not None
|
|
100
114
|
|
|
101
115
|
|
|
102
116
|
# ---------------------------------------------------------------------------
|
|
@@ -181,6 +181,20 @@ class SelfHealingOrchestrator:
|
|
|
181
181
|
with self._lock:
|
|
182
182
|
return set(self._excluded.keys())
|
|
183
183
|
|
|
184
|
+
def excluded_profiles(self) -> dict[str, str]:
|
|
185
|
+
"""Return a snapshot mapping excluded provider -> owning profile.
|
|
186
|
+
|
|
187
|
+
Used by the engine to re-arm recovery probes after a restart:
|
|
188
|
+
each excluded provider must be probed under the profile whose
|
|
189
|
+
``backend_health_threshold`` / ``recovery_probe_*`` settings
|
|
190
|
+
drove its exclusion. See :meth:`excluded_providers` for the
|
|
191
|
+
name-only variant.
|
|
192
|
+
"""
|
|
193
|
+
with self._lock:
|
|
194
|
+
return {
|
|
195
|
+
name: entry.profile for name, entry in self._excluded.items()
|
|
196
|
+
}
|
|
197
|
+
|
|
184
198
|
def reset(self) -> None:
|
|
185
199
|
"""Drop all state. Mainly for tests."""
|
|
186
200
|
with self._lock:
|
|
@@ -21,6 +21,8 @@ SDKs send values like "2023-06-01"; we log it for diagnostics only.
|
|
|
21
21
|
|
|
22
22
|
from __future__ import annotations
|
|
23
23
|
|
|
24
|
+
import asyncio
|
|
25
|
+
import contextlib
|
|
24
26
|
import json
|
|
25
27
|
from collections.abc import AsyncIterator
|
|
26
28
|
from typing import Any
|
|
@@ -36,6 +38,8 @@ from coderouter.routing import (
|
|
|
36
38
|
NoProvidersAvailableError,
|
|
37
39
|
)
|
|
38
40
|
from coderouter.routing.auto_router import RESERVED_PROFILE_NAME, classify
|
|
41
|
+
from coderouter.token_estimation import extract_text_from_anthropic_request
|
|
42
|
+
from coderouter.token_estimation_accurate import count_tokens, is_accuracy_available
|
|
39
43
|
from coderouter.translation import (
|
|
40
44
|
AnthropicRequest,
|
|
41
45
|
AnthropicStreamEvent,
|
|
@@ -51,6 +55,109 @@ _ANTHROPIC_BETA_HEADER = "anthropic-beta"
|
|
|
51
55
|
_CTX_BUDGET_HEADER = "X-CodeRouter-Context-Budget"
|
|
52
56
|
_DRIFT_HEADER = "X-CodeRouter-Drift"
|
|
53
57
|
|
|
58
|
+
# M14: overall SSE stream ceiling. A single streamed request can legitimately
|
|
59
|
+
# run much longer than one provider call (long generations, tool turns), so we
|
|
60
|
+
# derive the ceiling from the profile's per-call ``timeout_s`` scaled up, and
|
|
61
|
+
# fall back to a large default when no profile timeout is configured. This is a
|
|
62
|
+
# safety net against a wedged upstream holding a client (and the upstream
|
|
63
|
+
# socket) open forever — NOT a tight per-token deadline.
|
|
64
|
+
_STREAM_TIMEOUT_MULTIPLIER = 20.0
|
|
65
|
+
_STREAM_TIMEOUT_DEFAULT_S = 900.0
|
|
66
|
+
_STREAM_TIMEOUT_MIN_S = 60.0
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _resolve_stream_timeout_s(engine: FallbackEngine, profile: str | None) -> float:
|
|
70
|
+
"""M14: derive the overall stream ceiling (seconds) for a profile.
|
|
71
|
+
|
|
72
|
+
Uses the profile's per-call ``timeout_s`` (or the first provider's, or
|
|
73
|
+
the default) scaled by ``_STREAM_TIMEOUT_MULTIPLIER`` so a whole stream
|
|
74
|
+
gets a generous but bounded budget. Never returns below
|
|
75
|
+
``_STREAM_TIMEOUT_MIN_S``. Any resolution failure (stub config in tests,
|
|
76
|
+
missing profile) falls back to ``_STREAM_TIMEOUT_DEFAULT_S`` — the guard
|
|
77
|
+
stays active rather than disabling itself. Does NOT change the config
|
|
78
|
+
schema.
|
|
79
|
+
"""
|
|
80
|
+
per_call: float | None = None
|
|
81
|
+
try:
|
|
82
|
+
config = engine.config
|
|
83
|
+
chosen = profile or config.default_profile
|
|
84
|
+
chain_cfg = config.profile_by_name(chosen)
|
|
85
|
+
per_call = getattr(chain_cfg, "timeout_s", None)
|
|
86
|
+
if per_call is None:
|
|
87
|
+
# Fall back to the first provider's timeout.
|
|
88
|
+
for pname in getattr(chain_cfg, "providers", []) or []:
|
|
89
|
+
pconf = next(
|
|
90
|
+
(p for p in config.providers if p.name == pname), None
|
|
91
|
+
)
|
|
92
|
+
if pconf is not None:
|
|
93
|
+
per_call = getattr(pconf, "timeout_s", None)
|
|
94
|
+
break
|
|
95
|
+
except (AttributeError, KeyError, ValueError):
|
|
96
|
+
per_call = None
|
|
97
|
+
|
|
98
|
+
if per_call is None:
|
|
99
|
+
return _STREAM_TIMEOUT_DEFAULT_S
|
|
100
|
+
return max(_STREAM_TIMEOUT_MIN_S, float(per_call) * _STREAM_TIMEOUT_MULTIPLIER)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
async def _guard_stream(
|
|
104
|
+
source: AsyncIterator[str],
|
|
105
|
+
*,
|
|
106
|
+
timeout_s: float,
|
|
107
|
+
label: str,
|
|
108
|
+
) -> AsyncIterator[str]:
|
|
109
|
+
"""M14: enforce an overall timeout + clean cancellation on an SSE stream.
|
|
110
|
+
|
|
111
|
+
Wraps ``source`` (an iterator of already-formatted SSE frames) so that:
|
|
112
|
+
|
|
113
|
+
* The total wall-clock time is bounded by ``timeout_s`` via
|
|
114
|
+
``asyncio.timeout`` — a wedged upstream can no longer hold the client
|
|
115
|
+
(and the upstream socket) open indefinitely.
|
|
116
|
+
* On client disconnect (the ASGI server throws ``CancelledError`` into
|
|
117
|
+
the generator) or on timeout, the underlying ``source`` generator is
|
|
118
|
+
explicitly closed via ``aclose()``. Closing the engine generator runs
|
|
119
|
+
its ``finally`` blocks, which lets the adapter's ``httpx`` streaming
|
|
120
|
+
context manager exit and release the upstream connection instead of
|
|
121
|
+
leaking it until GC.
|
|
122
|
+
|
|
123
|
+
Errors raised by ``source`` itself (in-stream ``event: error`` framing)
|
|
124
|
+
are produced by the caller's iterator before it reaches here, so this
|
|
125
|
+
wrapper only has to deal with timeout / cancellation / normal completion.
|
|
126
|
+
"""
|
|
127
|
+
try:
|
|
128
|
+
async with asyncio.timeout(timeout_s):
|
|
129
|
+
async for frame in source:
|
|
130
|
+
yield frame
|
|
131
|
+
except TimeoutError:
|
|
132
|
+
# Overall ceiling hit. Emit a terminal error frame so a
|
|
133
|
+
# spec-compliant client sees a clean stream end rather than a
|
|
134
|
+
# silently truncated body, then stop.
|
|
135
|
+
logger.warning("sse-stream-timeout", extra={"label": label, "timeout_s": timeout_s})
|
|
136
|
+
err_event = AnthropicStreamEvent(
|
|
137
|
+
type="error",
|
|
138
|
+
data={
|
|
139
|
+
"type": "error",
|
|
140
|
+
"error": {
|
|
141
|
+
"type": "timeout_error",
|
|
142
|
+
"message": f"stream exceeded {timeout_s:.0f}s ceiling",
|
|
143
|
+
},
|
|
144
|
+
},
|
|
145
|
+
)
|
|
146
|
+
yield _format_anthropic_sse(err_event)
|
|
147
|
+
except asyncio.CancelledError:
|
|
148
|
+
# Client disconnected. Re-raise after ensuring the source is closed
|
|
149
|
+
# in the finally block so the upstream connection is released.
|
|
150
|
+
logger.info("sse-client-disconnect", extra={"label": label})
|
|
151
|
+
raise
|
|
152
|
+
finally:
|
|
153
|
+
# Guarantee the upstream generator is finalized on every exit path
|
|
154
|
+
# (normal completion, timeout, or cancellation).
|
|
155
|
+
aclose = getattr(source, "aclose", None)
|
|
156
|
+
if aclose is not None:
|
|
157
|
+
# Best-effort cleanup; never mask the original exit reason.
|
|
158
|
+
with contextlib.suppress(asyncio.CancelledError, Exception):
|
|
159
|
+
await aclose()
|
|
160
|
+
|
|
54
161
|
|
|
55
162
|
@router.post("/messages", response_model=None)
|
|
56
163
|
async def messages(
|
|
@@ -151,8 +258,17 @@ async def messages(
|
|
|
151
258
|
drift_severity = engine.last_drift_severity
|
|
152
259
|
if drift_severity:
|
|
153
260
|
stream_headers[_DRIFT_HEADER] = drift_severity
|
|
154
|
-
|
|
261
|
+
# M14: wrap the SSE iterator with an overall timeout + client
|
|
262
|
+
# disconnect cleanup so a wedged upstream cannot pin the client
|
|
263
|
+
# (or the upstream socket) open forever.
|
|
264
|
+
timeout_s = _resolve_stream_timeout_s(engine, anth_req.profile)
|
|
265
|
+
guarded = _guard_stream(
|
|
155
266
|
_anthropic_sse_iterator(engine, anth_req),
|
|
267
|
+
timeout_s=timeout_s,
|
|
268
|
+
label="anthropic",
|
|
269
|
+
)
|
|
270
|
+
return StreamingResponse(
|
|
271
|
+
guarded,
|
|
156
272
|
media_type="text/event-stream",
|
|
157
273
|
headers=stream_headers,
|
|
158
274
|
)
|
|
@@ -202,6 +318,112 @@ async def messages(
|
|
|
202
318
|
return anth_resp.model_dump(exclude_none=True)
|
|
203
319
|
|
|
204
320
|
|
|
321
|
+
def _resolve_count_tokens_tokenizer_path(
|
|
322
|
+
config: Any, profile: str | None
|
|
323
|
+
) -> str | None:
|
|
324
|
+
"""S1 (shim): best-effort resolve a local tokenizer.json for counting.
|
|
325
|
+
|
|
326
|
+
Routing for ``count_tokens`` is not a full chain resolution — we only
|
|
327
|
+
need *a* representative tokenizer for the profile the request would use.
|
|
328
|
+
We take the first provider of the resolved profile (or the default
|
|
329
|
+
profile) and return its ``tokenizer_path`` when declared. Any resolution
|
|
330
|
+
hiccup (unknown profile, empty chain, stub config in tests) returns
|
|
331
|
+
``None``, which makes :func:`count_tokens` fall back to the char/4
|
|
332
|
+
heuristic — a graceful degrade rather than an error.
|
|
333
|
+
"""
|
|
334
|
+
try:
|
|
335
|
+
chosen = profile or config.default_profile
|
|
336
|
+
chain_cfg = config.profile_by_name(chosen)
|
|
337
|
+
for pname in getattr(chain_cfg, "providers", []) or []:
|
|
338
|
+
pconf = next((p for p in config.providers if p.name == pname), None)
|
|
339
|
+
if pconf is not None:
|
|
340
|
+
tok = getattr(pconf, "tokenizer_path", None)
|
|
341
|
+
if tok:
|
|
342
|
+
return tok
|
|
343
|
+
# First provider resolved but declares no tokenizer — stop
|
|
344
|
+
# here (don't scan the whole chain for an unrelated one).
|
|
345
|
+
return None
|
|
346
|
+
except (AttributeError, KeyError, ValueError, TypeError):
|
|
347
|
+
return None
|
|
348
|
+
return None
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
@router.post("/messages/count_tokens", response_model=None)
|
|
352
|
+
async def count_tokens_route(
|
|
353
|
+
payload: dict[str, Any],
|
|
354
|
+
request: Request,
|
|
355
|
+
x_coderouter_profile: str | None = Header(default=None, alias=_PROFILE_HEADER),
|
|
356
|
+
x_coderouter_mode: str | None = Header(default=None, alias=_MODE_HEADER),
|
|
357
|
+
) -> dict[str, Any]:
|
|
358
|
+
"""Anthropic ``POST /v1/messages/count_tokens`` — local token estimate.
|
|
359
|
+
|
|
360
|
+
Mirrors Anthropic's count_tokens endpoint: the request body has the
|
|
361
|
+
same shape as ``/v1/messages`` (minus the ``max_tokens`` requirement)
|
|
362
|
+
and the response is ``{"input_tokens": N}``. CodeRouter answers this
|
|
363
|
+
entirely locally — there is no upstream round-trip — using the same
|
|
364
|
+
text-extraction the language-tax / context-budget guards use, feeding
|
|
365
|
+
an accurate local tokenizer when the routed provider declares one and
|
|
366
|
+
the ``accuracy`` extra is installed, otherwise the char/4 heuristic.
|
|
367
|
+
|
|
368
|
+
Validation is deliberately looser than :class:`AnthropicRequest` (no
|
|
369
|
+
``max_tokens`` needed): ``model`` and a non-empty ``messages`` list are
|
|
370
|
+
required, everything else optional. Profile selection follows the same
|
|
371
|
+
body > profile-header > mode-header precedence as ``/messages`` so the
|
|
372
|
+
tokenizer resolves against the provider the real request would hit.
|
|
373
|
+
"""
|
|
374
|
+
config = request.app.state.config
|
|
375
|
+
|
|
376
|
+
if not isinstance(payload, dict):
|
|
377
|
+
raise HTTPException(status_code=400, detail="request body must be a JSON object")
|
|
378
|
+
|
|
379
|
+
messages = payload.get("messages")
|
|
380
|
+
if not isinstance(messages, list) or not messages:
|
|
381
|
+
raise HTTPException(
|
|
382
|
+
status_code=400,
|
|
383
|
+
detail="'messages' is required and must be a non-empty list",
|
|
384
|
+
)
|
|
385
|
+
if "model" not in payload:
|
|
386
|
+
raise HTTPException(status_code=400, detail="'model' is required")
|
|
387
|
+
|
|
388
|
+
# Profile selection — body field wins over header, mode header last
|
|
389
|
+
# (same policy as /messages). Kept minimal: we only need the profile to
|
|
390
|
+
# pick a representative tokenizer, so an unknown mode/profile degrades
|
|
391
|
+
# to the default rather than 400'ing the count.
|
|
392
|
+
profile = payload.get("profile") or x_coderouter_profile
|
|
393
|
+
if profile is None and x_coderouter_mode:
|
|
394
|
+
try:
|
|
395
|
+
profile = config.resolve_mode(x_coderouter_mode)
|
|
396
|
+
except (KeyError, AttributeError):
|
|
397
|
+
profile = None
|
|
398
|
+
|
|
399
|
+
# Combine system + messages (+ tool JSON length) into one text blob and
|
|
400
|
+
# count. tools contribute their JSON length as a coarse proxy for the
|
|
401
|
+
# schema tokens Anthropic would bill.
|
|
402
|
+
text = extract_text_from_anthropic_request(
|
|
403
|
+
system=payload.get("system"),
|
|
404
|
+
messages=messages,
|
|
405
|
+
)
|
|
406
|
+
tools = payload.get("tools")
|
|
407
|
+
if isinstance(tools, list) and tools:
|
|
408
|
+
with contextlib.suppress(TypeError, ValueError):
|
|
409
|
+
text = f"{text}\n{json.dumps(tools, ensure_ascii=False)}"
|
|
410
|
+
|
|
411
|
+
tokenizer_path = _resolve_count_tokens_tokenizer_path(config, profile)
|
|
412
|
+
input_tokens = count_tokens(text, tokenizer_path=tokenizer_path)
|
|
413
|
+
# Report the method count_tokens *actually* used: the accurate backend
|
|
414
|
+
# only engages when a path is declared AND the optional ``tokenizers``
|
|
415
|
+
# dependency is importable (see token_estimation_accurate). Otherwise
|
|
416
|
+
# count_tokens transparently falls back to char/4 — so we must too, or
|
|
417
|
+
# the log would misreport a heuristic count as tokenizer-accurate.
|
|
418
|
+
method = "tokenizer" if (tokenizer_path and is_accuracy_available()) else "heuristic"
|
|
419
|
+
|
|
420
|
+
logger.info(
|
|
421
|
+
"count-tokens-served",
|
|
422
|
+
extra={"method": method, "input_tokens": input_tokens},
|
|
423
|
+
)
|
|
424
|
+
return {"input_tokens": input_tokens}
|
|
425
|
+
|
|
426
|
+
|
|
205
427
|
async def _anthropic_sse_iterator(
|
|
206
428
|
engine: FallbackEngine, anth_req: AnthropicRequest
|
|
207
429
|
) -> AsyncIterator[str]:
|
coderouter/ingress/app.py
CHANGED
|
@@ -7,7 +7,10 @@ import os
|
|
|
7
7
|
from collections.abc import AsyncIterator
|
|
8
8
|
from contextlib import asynccontextmanager
|
|
9
9
|
|
|
10
|
-
from fastapi import FastAPI
|
|
10
|
+
from fastapi import FastAPI, Request
|
|
11
|
+
from fastapi.responses import JSONResponse
|
|
12
|
+
from starlette.middleware.base import BaseHTTPMiddleware
|
|
13
|
+
from starlette.types import ASGIApp
|
|
11
14
|
|
|
12
15
|
from coderouter import __version__
|
|
13
16
|
from coderouter.config import load_config
|
|
@@ -25,6 +28,137 @@ from coderouter.routing.capability import check_claude_code_chain_suitability
|
|
|
25
28
|
logger = get_logger(__name__)
|
|
26
29
|
|
|
27
30
|
|
|
31
|
+
# ---------------------------------------------------------------------------
|
|
32
|
+
# H8: DNS-rebinding protection via Host-header validation
|
|
33
|
+
# ---------------------------------------------------------------------------
|
|
34
|
+
|
|
35
|
+
# Loopback hosts that are always allowed. ``testserver`` is the default Host
|
|
36
|
+
# that Starlette's TestClient sends, so it is whitelisted to keep the existing
|
|
37
|
+
# suite (and any local integration tests) working without extra configuration.
|
|
38
|
+
_DEFAULT_ALLOWED_HOSTS: frozenset[str] = frozenset(
|
|
39
|
+
{"localhost", "127.0.0.1", "[::1]", "::1", "testserver"}
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _parse_allowed_hosts(raw: str | None) -> frozenset[str]:
|
|
44
|
+
"""Merge the built-in loopback set with ``CODEROUTER_ALLOWED_HOSTS``.
|
|
45
|
+
|
|
46
|
+
The env var is a comma-separated list of extra hostnames (no port) that
|
|
47
|
+
should be accepted — the escape hatch for deliberate external exposure.
|
|
48
|
+
"""
|
|
49
|
+
extra = {
|
|
50
|
+
part.strip().lower()
|
|
51
|
+
for part in (raw or "").split(",")
|
|
52
|
+
if part.strip()
|
|
53
|
+
}
|
|
54
|
+
return _DEFAULT_ALLOWED_HOSTS | frozenset(extra)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _host_without_port(host_header: str) -> str:
|
|
58
|
+
"""Strip the optional ``:port`` suffix from a Host header value.
|
|
59
|
+
|
|
60
|
+
Handles bracketed IPv6 literals (``[::1]:8080`` → ``[::1]``) as well as
|
|
61
|
+
the common ``host:port`` form. A bare ``[::1]`` or ``host`` is returned
|
|
62
|
+
unchanged.
|
|
63
|
+
"""
|
|
64
|
+
value = host_header.strip().lower()
|
|
65
|
+
if value.startswith("["):
|
|
66
|
+
# IPv6 literal: keep everything up to and including the closing bracket.
|
|
67
|
+
end = value.find("]")
|
|
68
|
+
if end != -1:
|
|
69
|
+
return value[: end + 1]
|
|
70
|
+
return value
|
|
71
|
+
# IPv4 / hostname: split off a trailing :port if present.
|
|
72
|
+
if ":" in value:
|
|
73
|
+
return value.rsplit(":", 1)[0]
|
|
74
|
+
return value
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
# ---------------------------------------------------------------------------
|
|
78
|
+
# M14: request body size limit (DoS protection)
|
|
79
|
+
# ---------------------------------------------------------------------------
|
|
80
|
+
|
|
81
|
+
# Default cap for incoming request bodies. Anthropic/OpenAI chat payloads are
|
|
82
|
+
# comfortably below this even with large system prompts; the ceiling exists to
|
|
83
|
+
# stop a client from streaming an unbounded body and exhausting memory.
|
|
84
|
+
_DEFAULT_MAX_BODY_BYTES = 64 * 1024 * 1024 # 64 MB
|
|
85
|
+
_MAX_BODY_BYTES_ENV = "CODEROUTER_MAX_BODY_BYTES"
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _parse_max_body_bytes(raw: str | None) -> int:
|
|
89
|
+
"""Resolve the body-size cap from ``CODEROUTER_MAX_BODY_BYTES``.
|
|
90
|
+
|
|
91
|
+
Falls back to :data:`_DEFAULT_MAX_BODY_BYTES` when the env var is unset,
|
|
92
|
+
empty, non-numeric, or non-positive — a misconfigured value must never
|
|
93
|
+
silently disable the guard.
|
|
94
|
+
"""
|
|
95
|
+
if raw is None or not raw.strip():
|
|
96
|
+
return _DEFAULT_MAX_BODY_BYTES
|
|
97
|
+
try:
|
|
98
|
+
value = int(raw.strip())
|
|
99
|
+
except ValueError:
|
|
100
|
+
return _DEFAULT_MAX_BODY_BYTES
|
|
101
|
+
return value if value > 0 else _DEFAULT_MAX_BODY_BYTES
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class BodySizeLimitMiddleware(BaseHTTPMiddleware):
|
|
105
|
+
"""Reject requests whose declared body exceeds the configured cap.
|
|
106
|
+
|
|
107
|
+
The check is on the incoming ``Content-Length`` header, so it is cheap
|
|
108
|
+
(no body is read) and fires before the route handler runs. Streaming
|
|
109
|
+
*responses* (SSE) are unaffected — this only inspects the request side.
|
|
110
|
+
A body over the limit fails closed with 413.
|
|
111
|
+
"""
|
|
112
|
+
|
|
113
|
+
def __init__(self, app: ASGIApp, max_bytes: int) -> None:
|
|
114
|
+
super().__init__(app)
|
|
115
|
+
self._max_bytes = max_bytes
|
|
116
|
+
|
|
117
|
+
async def dispatch(self, request: Request, call_next): # type: ignore[no-untyped-def]
|
|
118
|
+
content_length = request.headers.get("content-length")
|
|
119
|
+
if content_length is not None:
|
|
120
|
+
try:
|
|
121
|
+
declared = int(content_length)
|
|
122
|
+
except ValueError:
|
|
123
|
+
declared = -1
|
|
124
|
+
if declared > self._max_bytes:
|
|
125
|
+
return JSONResponse(
|
|
126
|
+
status_code=413,
|
|
127
|
+
content={
|
|
128
|
+
"detail": (
|
|
129
|
+
f"Request body too large: {declared} bytes exceeds "
|
|
130
|
+
f"the {self._max_bytes}-byte limit."
|
|
131
|
+
)
|
|
132
|
+
},
|
|
133
|
+
)
|
|
134
|
+
return await call_next(request)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
class HostValidationMiddleware(BaseHTTPMiddleware):
|
|
138
|
+
"""Reject requests whose Host header is not an allow-listed hostname.
|
|
139
|
+
|
|
140
|
+
This blocks DNS-rebinding attacks: even when CodeRouter is bound to
|
|
141
|
+
``127.0.0.1``, a malicious page can point a hostname it controls at the
|
|
142
|
+
loopback address and drive the local API from the victim's browser. By
|
|
143
|
+
pinning the accepted Host values we make such cross-origin requests fail
|
|
144
|
+
closed with 403.
|
|
145
|
+
"""
|
|
146
|
+
|
|
147
|
+
def __init__(self, app: ASGIApp, allowed_hosts: frozenset[str]) -> None:
|
|
148
|
+
super().__init__(app)
|
|
149
|
+
self._allowed = allowed_hosts
|
|
150
|
+
|
|
151
|
+
async def dispatch(self, request: Request, call_next): # type: ignore[no-untyped-def]
|
|
152
|
+
host_header = request.headers.get("host", "")
|
|
153
|
+
host = _host_without_port(host_header)
|
|
154
|
+
if host not in self._allowed:
|
|
155
|
+
return JSONResponse(
|
|
156
|
+
status_code=403,
|
|
157
|
+
content={"detail": f"Host {host_header!r} is not allowed."},
|
|
158
|
+
)
|
|
159
|
+
return await call_next(request)
|
|
160
|
+
|
|
161
|
+
|
|
28
162
|
def create_app(config_path: str | None = None) -> FastAPI:
|
|
29
163
|
"""Build a FastAPI app with routes, engine, and lifespan installed.
|
|
30
164
|
|
|
@@ -189,6 +323,15 @@ def create_app(config_path: str | None = None) -> FastAPI:
|
|
|
189
323
|
with contextlib.suppress(Exception):
|
|
190
324
|
await engine.shutdown_recovery_probes()
|
|
191
325
|
|
|
326
|
+
# H3: close each adapter's shared httpx.AsyncClient so pooled
|
|
327
|
+
# connections / keep-alive sockets are released cleanly instead of
|
|
328
|
+
# being left to garbage collection. Adapters are cached on the
|
|
329
|
+
# engine (one per provider); ``aclose`` is idempotent and a no-op
|
|
330
|
+
# when the adapter never issued a request.
|
|
331
|
+
for _adapter in engine._adapters.values():
|
|
332
|
+
with contextlib.suppress(Exception):
|
|
333
|
+
await _adapter.aclose()
|
|
334
|
+
|
|
192
335
|
# v2.0-K: persist state and close audit log on shutdown.
|
|
193
336
|
if state_store is not None:
|
|
194
337
|
with contextlib.suppress(Exception):
|
|
@@ -228,6 +371,24 @@ def create_app(config_path: str | None = None) -> FastAPI:
|
|
|
228
371
|
app.state.engine = engine
|
|
229
372
|
app.state.config = config
|
|
230
373
|
|
|
374
|
+
# H8: DNS-rebinding protection. Applied to every route so a hostname an
|
|
375
|
+
# attacker controls cannot be pointed at loopback and used to drive the
|
|
376
|
+
# local API from a victim's browser. Extra hostnames for deliberate
|
|
377
|
+
# external exposure come from CODEROUTER_ALLOWED_HOSTS (comma-separated).
|
|
378
|
+
allowed_hosts = _parse_allowed_hosts(
|
|
379
|
+
os.environ.get("CODEROUTER_ALLOWED_HOSTS")
|
|
380
|
+
)
|
|
381
|
+
app.add_middleware(HostValidationMiddleware, allowed_hosts=allowed_hosts)
|
|
382
|
+
|
|
383
|
+
# M14: request body size limit (DoS protection). Added after the Host
|
|
384
|
+
# middleware so that — because add_middleware wraps outermost-last — Host
|
|
385
|
+
# validation runs first and the size guard second. SSE streaming responses
|
|
386
|
+
# are unaffected: the check only reads the request Content-Length.
|
|
387
|
+
max_body_bytes = _parse_max_body_bytes(
|
|
388
|
+
os.environ.get(_MAX_BODY_BYTES_ENV)
|
|
389
|
+
)
|
|
390
|
+
app.add_middleware(BodySizeLimitMiddleware, max_bytes=max_body_bytes)
|
|
391
|
+
|
|
231
392
|
@app.get("/healthz")
|
|
232
393
|
async def healthz() -> dict[str, object]:
|
|
233
394
|
"""Lightweight liveness / config snapshot endpoint.
|