coderouter-cli 2.0.0__py3-none-any.whl → 2.1.0__py3-none-any.whl

coderouter/guards/drift_detection.py

@@ -0,0 +1,308 @@
+"""Drift detection guard (v2.0-G, L4).
+
+Detects gradual quality degradation in model responses during long-running
+agent sessions. Unlike L5 (binary crash) and adaptive routing (latency),
+drift detection targets the "succeeds but quality is decaying" pattern:
+empty responses, shrinking output length, tool-call silence, anomalous
+stop reasons.
+
+Architecture
+============
+
+Three layers:
+
+  1. **Observation model** — :class:`ResponseObservation` captures the
+     quality-relevant fields from each successful provider response.
+  2. **Detector** — :func:`detect_drift` is a pure function that takes
+     a window of observations and thresholds, returns a
+     :class:`DriftVerdict`.
+  3. **Window manager** — :class:`DriftWindow` maintains per-provider
+     rolling deques of observations, thread-safe for the async engine.
+
+The engine calls :meth:`DriftWindow.record` after each provider-ok/failed
+event, then calls :func:`detect_drift` to check whether corrective action
+is needed.
+
+Signals
+=======
+
+  * ``empty_response_rate`` — fraction of responses with output_tokens == 0
+  * ``length_collapse`` — median output_tokens in the recent half vs. the
+    earlier half of the window; ratio below threshold = collapse
+  * ``tool_silence_rate`` — fraction of responses missing tool_use blocks
+    (only meaningful when the request contained tools)
+  * ``stop_anomaly_rate`` — fraction of responses with unexpected stop_reason
+    (not "end_turn" / "tool_use" / "max_tokens")
+  * ``error_rate`` — fraction of attempts that ended in failure
+
+Thresholds are bundled as :class:`DriftThresholds` with three presets
+(``low`` / ``normal`` / ``high`` sensitivity).
+"""
+
+from __future__ import annotations
+
+import statistics
+from collections import deque
+from dataclasses import dataclass, field
+from typing import Literal
+
+# ---------------------------------------------------------------------------
+# Observation model
+# ---------------------------------------------------------------------------
+
+
+@dataclass(frozen=True, slots=True)
+class ResponseObservation:
+    """Quality-relevant snapshot of a single provider response.
+
+    Captured post-response (after adapter translation), before returning
+    to the client. Fields are intentionally minimal — only what the
+    detector needs.
+    """
+
+    provider: str
+    output_tokens: int
+    has_tool_use: bool
+    """Whether the response contained at least one tool_use block."""
+    request_had_tools: bool
+    """Whether the request included a tools[] array (context for tool_silence)."""
+    stop_reason: str | None
+    """Anthropic stop_reason: 'end_turn' / 'tool_use' / 'max_tokens' / None."""
+    is_error: bool = False
+    """True if the attempt ended in provider-failed / provider-failed-midstream."""
+    stream: bool = False
+
+
+# ---------------------------------------------------------------------------
+# Thresholds
+# ---------------------------------------------------------------------------
+
+
+@dataclass(frozen=True, slots=True)
+class DriftThresholds:
+    """Threshold set for drift detection.
+
+    Each field is the value above which (or below which for length_collapse)
+    the corresponding signal is considered anomalous.
+    """
+
+    # Rate thresholds (signal > threshold → anomaly)
+    empty_response_rate: float = 0.3
+    """Fraction of responses with output_tokens == 0 to trigger."""
+    stop_anomaly_rate: float = 0.4
+    """Fraction of responses with unexpected stop_reason."""
+    error_rate: float = 0.25
+    """Fraction of failed attempts."""
+    tool_silence_rate: float = 0.7
+    """Fraction of tool-eligible responses missing tool_use."""
+
+    # Ratio threshold (recent_median / earlier_median < threshold → collapse)
+    length_collapse_ratio: float = 0.5
+    """If recent half median is < 50% of earlier half median → collapse."""
+
+    # Minimum observations before detection fires
+    min_window_fill: int = 6
+    """Don't trigger until at least this many observations in the window."""
+
+
+# Presets
+THRESHOLDS_LOW = DriftThresholds(
+    empty_response_rate=0.5,
+    length_collapse_ratio=0.3,
+    tool_silence_rate=0.8,
+    stop_anomaly_rate=0.6,
+    error_rate=0.4,
+    min_window_fill=10,
+)
+
+THRESHOLDS_NORMAL = DriftThresholds()  # defaults
+
+THRESHOLDS_HIGH = DriftThresholds(
+    empty_response_rate=0.2,
+    length_collapse_ratio=0.7,
+    tool_silence_rate=0.5,
+    stop_anomaly_rate=0.3,
+    error_rate=0.15,
+    min_window_fill=4,
+)
+
+SENSITIVITY_PRESETS: dict[str, DriftThresholds] = {
+    "low": THRESHOLDS_LOW,
+    "normal": THRESHOLDS_NORMAL,
+    "high": THRESHOLDS_HIGH,
+}
+
+
+# ---------------------------------------------------------------------------
+# Verdict
+# ---------------------------------------------------------------------------
+
+
+@dataclass(frozen=True, slots=True)
+class DriftVerdict:
+    """Result of drift detection for a single provider window."""
+
+    drifted: bool
+    severity: Literal["none", "mild", "severe"]
+    signals: dict[str, float]
+    """Signal name → computed value for observability."""
+    reason: str
+    """Human-readable explanation of why drift was detected (empty if none)."""
+
+
+_NO_DRIFT = DriftVerdict(drifted=False, severity="none", signals={}, reason="")
+
+
+# ---------------------------------------------------------------------------
+# Detector (pure function)
+# ---------------------------------------------------------------------------
+
+
+def detect_drift(
+    window: list[ResponseObservation],
+    thresholds: DriftThresholds | None = None,
+) -> DriftVerdict:
+    """Analyze a window of observations and return a drift verdict.
+
+    Pure function — no I/O, no side effects. Safe to call from any context.
+
+    Parameters
+    ----------
+    window:
+        List of recent :class:`ResponseObservation` for a single provider,
+        ordered oldest-first.
+    thresholds:
+        Detection thresholds. Defaults to ``THRESHOLDS_NORMAL``.
+
+    Returns
+    -------
+    DriftVerdict with severity ``none`` / ``mild`` / ``severe``.
+    """
+    if thresholds is None:
+        thresholds = THRESHOLDS_NORMAL
+
+    if len(window) < thresholds.min_window_fill:
+        return _NO_DRIFT
+
+    signals: dict[str, float] = {}
+    mild_flags: list[str] = []
+    severe_flags: list[str] = []
+
+    total = len(window)
+
+    # --- Signal 1: Empty response rate ---
+    empty_count = sum(1 for obs in window if obs.output_tokens == 0 and not obs.is_error)
+    non_error_count = sum(1 for obs in window if not obs.is_error)
+    if non_error_count > 0:
+        empty_rate = empty_count / non_error_count
+        signals["empty_response_rate"] = round(empty_rate, 3)
+        if empty_rate > thresholds.empty_response_rate:
+            severe_flags.append(f"empty_response_rate={empty_rate:.2f}")
+
+    # --- Signal 2: Length collapse (median comparison) ---
+    non_error_lengths = [obs.output_tokens for obs in window if not obs.is_error]
+    if len(non_error_lengths) >= 4:
+        mid = len(non_error_lengths) // 2
+        earlier_half = non_error_lengths[:mid]
+        recent_half = non_error_lengths[mid:]
+        earlier_median = statistics.median(earlier_half)
+        recent_median = statistics.median(recent_half)
+        if earlier_median > 0:
+            collapse_ratio = recent_median / earlier_median
+            signals["length_collapse_ratio"] = round(collapse_ratio, 3)
+            if collapse_ratio < thresholds.length_collapse_ratio:
+                severe_flags.append(
+                    f"length_collapse={collapse_ratio:.2f}"
+                    f" (recent_median={recent_median:.0f}, earlier={earlier_median:.0f})"
+                )
+
+    # --- Signal 3: Tool silence rate ---
+    tool_eligible = [obs for obs in window if obs.request_had_tools and not obs.is_error]
+    if len(tool_eligible) >= 3:
+        tool_silent_count = sum(1 for obs in tool_eligible if not obs.has_tool_use)
+        tool_silence_rate = tool_silent_count / len(tool_eligible)
+        signals["tool_silence_rate"] = round(tool_silence_rate, 3)
+        if tool_silence_rate > thresholds.tool_silence_rate:
+            mild_flags.append(f"tool_silence_rate={tool_silence_rate:.2f}")
+
+    # --- Signal 4: Stop reason anomaly rate ---
+    _EXPECTED_STOP = {"end_turn", "tool_use", "max_tokens"}
+    non_error_obs = [obs for obs in window if not obs.is_error]
+    if non_error_obs:
+        anomaly_count = sum(
+            1 for obs in non_error_obs if obs.stop_reason not in _EXPECTED_STOP
+        )
+        stop_anomaly_rate = anomaly_count / len(non_error_obs)
+        signals["stop_anomaly_rate"] = round(stop_anomaly_rate, 3)
+        if stop_anomaly_rate > thresholds.stop_anomaly_rate:
+            mild_flags.append(f"stop_anomaly_rate={stop_anomaly_rate:.2f}")
+
+    # --- Signal 5: Error rate ---
+    error_count = sum(1 for obs in window if obs.is_error)
+    error_rate = error_count / total
+    signals["error_rate"] = round(error_rate, 3)
+    if error_rate > thresholds.error_rate:
+        mild_flags.append(f"error_rate={error_rate:.2f}")
+
+    # --- Severity synthesis ---
+    if severe_flags:
+        severity: Literal["none", "mild", "severe"] = "severe"
+    elif len(mild_flags) >= 2:
+        severity = "severe"
+    elif mild_flags:
+        severity = "mild"
+    else:
+        return DriftVerdict(drifted=False, severity="none", signals=signals, reason="")
+
+    reason_parts = severe_flags + mild_flags
+    return DriftVerdict(
+        drifted=True,
+        severity=severity,
+        signals=signals,
+        reason=", ".join(reason_parts),
+    )
+
+
+# ---------------------------------------------------------------------------
+# Window manager
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class DriftWindow:
+    """Per-provider rolling window of response observations.
+
+    Thread-safe for the single-threaded async event loop (no locking needed
+    since all access happens on the same asyncio loop). If CodeRouter ever
+    goes multi-threaded, add a Lock.
+    """
+
+    max_size: int = 20
+    _windows: dict[str, deque[ResponseObservation]] = field(default_factory=dict)
+
+    def record(self, obs: ResponseObservation) -> None:
+        """Append an observation to the provider's window."""
+        dq = self._windows.get(obs.provider)
+        if dq is None:
+            dq = deque(maxlen=self.max_size)
+            self._windows[obs.provider] = dq
+        dq.append(obs)
+
+    def get_window(self, provider: str) -> list[ResponseObservation]:
+        """Return a snapshot of the provider's window (oldest-first)."""
+        dq = self._windows.get(provider)
+        if dq is None:
+            return []
+        return list(dq)
+
+    def clear(self, provider: str) -> None:
+        """Clear a provider's window (e.g. after recovery)."""
+        self._windows.pop(provider, None)
+
+    def clear_all(self) -> None:
+        """Reset all windows."""
+        self._windows.clear()
+
+    def __len__(self) -> int:
+        """Total observations across all providers."""
+        return sum(len(dq) for dq in self._windows.values())

coderouter/ingress/anthropic_routes.py

@@ -49,6 +49,7 @@ _MODE_HEADER = "x-coderouter-mode"
 _ANTHROPIC_VERSION_HEADER = "anthropic-version"
 _ANTHROPIC_BETA_HEADER = "anthropic-beta"
 _CTX_BUDGET_HEADER = "X-CodeRouter-Context-Budget"
+_DRIFT_HEADER = "X-CodeRouter-Drift"
 
 
 @router.post("/messages", response_model=None)
@@ -144,6 +145,12 @@ async def messages(
         }
         if ctx_budget_status:
             stream_headers[_CTX_BUDGET_HEADER] = ctx_budget_status
+        # v2.0-G: drift header is set post-stream via a trailer-like
+        # mechanism — for streaming we cannot know the verdict before
+        # the first chunk ships. Instead, check pre-existing drift state.
+        drift_severity = engine.last_drift_severity
+        if drift_severity:
+            stream_headers[_DRIFT_HEADER] = drift_severity
         return StreamingResponse(
             _anthropic_sse_iterator(engine, anth_req),
             media_type="text/event-stream",
@@ -167,10 +174,18 @@ async def messages(
             detail=_tool_loop_break_detail(exc),
         ) from exc
 
+    # v2.0-G: collect drift header after engine dispatch.
+    drift_severity = engine.last_drift_severity
+    resp_headers: dict[str, str] = {}
     if ctx_budget_status:
+        resp_headers[_CTX_BUDGET_HEADER] = ctx_budget_status
+    if drift_severity:
+        resp_headers[_DRIFT_HEADER] = drift_severity
+
+    if resp_headers:
         return JSONResponse(
             content=anth_resp.model_dump(exclude_none=True),
-            headers={_CTX_BUDGET_HEADER: ctx_budget_status},
+            headers=resp_headers,
         )
     return anth_resp.model_dump(exclude_none=True)
 
@@ -233,17 +248,66 @@ async def _anthropic_sse_iterator(
             "sse-midstream-error",
             extra={"provider": exc.provider, "original": str(exc.original)},
         )
-        err_event = AnthropicStreamEvent(
-            type="error",
-            data={
-                "type": "error",
-                "error": {
-                    "type": "api_error",
-                    "message": str(exc),
+
+        # v2.0-H (L6): partial stitch surface mode — synthesize a graceful
+        # stream termination that delivers accumulated text to the client.
+        profile_name = anth_req.profile or engine.config.default_profile
+        partial_action = "off"
+        try:
+            chain_cfg = engine.config.profile_by_name(profile_name)
+            partial_action = chain_cfg.partial_stitch_action
+        except (KeyError, ValueError):
+            pass
+
+        if partial_action == "surface" and exc.partial_content:
+            # Emit message_delta with accumulated usage (signals stream end).
+            yield _format_anthropic_sse(AnthropicStreamEvent(
+                type="message_delta",
+                data={
+                    "type": "message_delta",
+                    "delta": {"stop_reason": None, "stop_sequence": None},
+                    "usage": {"output_tokens": 0},
                 },
-            },
-        )
-        yield _format_anthropic_sse(err_event)
+            ))
+            # Emit message_stop so the client sees a complete stream.
+            yield _format_anthropic_sse(AnthropicStreamEvent(
+                type="message_stop",
+                data={"type": "message_stop"},
+            ))
+            # Emit coderouter_partial metadata event (client-optional).
+            yield _format_anthropic_sse(AnthropicStreamEvent(
+                type="coderouter_partial",
+                data={
+                    "type": "coderouter_partial",
+                    "partial_content": exc.partial_content,
+                    "provider": exc.provider,
+                    "reason": "mid_stream_failure",
+                    "original_error": str(exc.original)[:200],
+                },
+            ))
+            logger.info(
+                "partial-stitch-surfaced",
+                extra={
+                    "provider": exc.provider,
+                    "profile": profile_name,
+                    "text_blocks": len(exc.partial_content),
+                    "text_length": sum(
+                        len(b.get("text", "")) for b in exc.partial_content
+                    ),
+                },
+            )
+        else:
+            err_event = AnthropicStreamEvent(
+                type="error",
+                data={
+                    "type": "error",
+                    "error": {
+                        "type": "api_error",
+                        "message": str(exc),
+                    },
+                },
+            )
+            yield _format_anthropic_sse(err_event)
 
 
 def _format_anthropic_sse(ev: AnthropicStreamEvent) -> str:

coderouter/ingress/app.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import contextlib
 import os
 from collections.abc import AsyncIterator
 from contextlib import asynccontextmanager
@@ -71,7 +72,45 @@ def create_app(config_path: str | None = None) -> FastAPI:
         # chronological order. Non-fatal — the chain still works, just
         # potentially sub-optimally for the agentic harness.
         check_claude_code_chain_suitability(config, logger=logger)
+
+        # v2.0-I: launch continuous probe background task if configured.
+        probe_task = None
+        shutdown_event = None
+        if config.continuous_probe == "active":
+            import asyncio
+
+            from coderouter.guards.continuous_probe import probe_loop
+            from coderouter.routing.capability import get_default_registry
+
+            shutdown_event = asyncio.Event()
+            probe_task = asyncio.create_task(
+                probe_loop(
+                    config.providers,
+                    record_fn=engine.backend_health.record_attempt,
+                    interval_s=config.probe_interval_s,
+                    timeout_s=config.probe_timeout_s,
+                    probe_paid=config.probe_paid,
+                    shutdown_event=shutdown_event,
+                    registry=get_default_registry(),
+                )
+            )
+            logger.info(
+                "continuous-probe-started",
+                extra={
+                    "interval_s": config.probe_interval_s,
+                    "probe_paid": config.probe_paid,
+                    "providers": len(config.providers),
+                },
+            )
+
         yield
+
+        # Graceful shutdown of probe task
+        if probe_task is not None and shutdown_event is not None:
+            shutdown_event.set()
+            with contextlib.suppress(Exception):
+                await probe_task
+
         logger.info("coderouter-shutdown")
 
     app = FastAPI(