coderouter-cli 2.0.0__py3-none-any.whl → 2.1.0__py3-none-any.whl

coderouter/logging.py

@@ -1101,3 +1101,265 @@ def log_context_budget_trimmed(
         "max_context_tokens": max_context_tokens,
     }
     logger.info("context-budget-trimmed", extra=payload)
+
+
+# ---------------------------------------------------------------------------
+# v2.0-G (L4): Drift detection logging
+# ---------------------------------------------------------------------------
+
+
+def log_drift_detected(
+    logger: logging.Logger,
+    *,
+    provider: str,
+    profile: str,
+    severity: str,
+    reason: str,
+    action: str,
+    signals: dict[str, float],
+) -> None:
+    """Emit a ``drift-detected`` warning line.
+
+    Fired when the drift detector finds quality degradation in the
+    provider's rolling response window.
+    """
+    logger.warning(
+        "drift-detected",
+        extra={
+            "provider": provider,
+            "profile": profile,
+            "severity": severity,
+            "reason": reason,
+            "action": action,
+            "signals": signals,
+        },
+    )
+
+
+def log_drift_promoted(
+    logger: logging.Logger,
+    *,
+    provider: str,
+    profile: str,
+    demoted_to_rank: int,
+    cooldown_s: int,
+) -> None:
+    """Emit a ``drift-promoted`` info line.
+
+    Fired when a drifted provider is demoted in the chain and a
+    different provider takes over as primary.
+    """
+    logger.info(
+        "drift-promoted",
+        extra={
+            "provider": provider,
+            "profile": profile,
+            "demoted_to_rank": demoted_to_rank,
+            "cooldown_s": cooldown_s,
+        },
+    )
+
+
+def log_drift_reload_attempted(
+    logger: logging.Logger,
+    *,
+    provider: str,
+    success: bool,
+) -> None:
+    """Emit a ``drift-reload-attempted`` info line.
+
+    Fired after attempting an Ollama KV cache flush (keep_alive=0).
+    """
+    logger.info(
+        "drift-reload-attempted",
+        extra={
+            "provider": provider,
+            "success": success,
+        },
+    )
+
+
+def log_drift_recovered(
+    logger: logging.Logger,
+    *,
+    provider: str,
+    profile: str,
+    after_s: float,
+) -> None:
+    """Emit a ``drift-recovered`` info line.
+
+    Fired when a previously-drifted provider's cooldown expires and its
+    rank is restored.
+    """
+    logger.info(
+        "drift-recovered",
+        extra={
+            "provider": provider,
+            "profile": profile,
+            "after_s": round(after_s, 1),
+        },
+    )
+
+
+# ---------------------------------------------------------------------------
+# v2.0-H (L6): Partial stitch surfaced logging
+# ---------------------------------------------------------------------------
+
+
+def log_partial_stitch_surfaced(
+    logger: logging.Logger,
+    *,
+    provider: str,
+    profile: str,
+    text_blocks: int,
+    text_length: int,
+) -> None:
+    """Emit a ``partial-stitch-surfaced`` info line.
+
+    Fired when a mid-stream failure is gracefully terminated with partial
+    content delivered to the client (partial_stitch_action=surface).
+    """
+    logger.info(
+        "partial-stitch-surfaced",
+        extra={
+            "provider": provider,
+            "profile": profile,
+            "text_blocks": text_blocks,
+            "text_length": text_length,
+        },
+    )
+
+
+# ---------------------------------------------------------------------------
+# v2.0-I: Continuous probe log shapes
+#
+# Two event lanes mirror the backend-health triplet:
+#   * ``probe-completed``       — info: a single provider probe finished
+#                                  (success or failure). Quiet in normal
+#                                  operation; operators grep for these to
+#                                  diagnose individual backend issues.
+#   * ``probe-round-completed`` — info: one full sweep across all probed
+#                                  providers finished. Summary counter for
+#                                  dashboards to render "probes/min" and
+#                                  "failure ratio per round".
+# ---------------------------------------------------------------------------
+
+
+class ProbeCompletedPayload(TypedDict):
+    """Structured shape of the ``probe-completed`` log record.
+
+    Fields
+        provider: the provider that was probed.
+        success: whether the 1-token probe request succeeded.
+        latency_ms: round-trip time of the probe in milliseconds.
+        error: short error message (truncated) when success=False;
+            None on success.
+        model_name: model name extracted from the probe response,
+            if available.
+    """
+
+    provider: str
+    success: bool
+    latency_ms: float
+    error: str | None
+    model_name: str | None
+
+
+class ProbeRoundCompletedPayload(TypedDict):
+    """Structured shape of the ``probe-round-completed`` log record.
+
+    Fields
+        providers_probed: number of providers probed in this round.
+        failures: number of probe failures in this round.
+    """
+
+    providers_probed: int
+    failures: int
+
+
+def log_probe_completed(
+    logger: logging.Logger,
+    *,
+    provider: str,
+    success: bool,
+    latency_ms: float,
+    error: str | None = None,
+    model_name: str | None = None,
+) -> None:
+    """Emit a ``probe-completed`` info line for one provider probe.
+
+    Single chokepoint mirroring :func:`log_capability_degraded`. Info
+    level — individual probes are diagnostic noise at normal operation;
+    operators filter on ``success=False`` for alerting.
+    """
+    payload: ProbeCompletedPayload = {
+        "provider": provider,
+        "success": success,
+        "latency_ms": round(latency_ms, 1),
+        "error": error,
+        "model_name": model_name,
+    }
+    logger.info("probe-completed", extra=payload)
+
+
+def log_probe_round_completed(
+    logger: logging.Logger,
+    *,
+    providers_probed: int,
+    failures: int,
+) -> None:
+    """Emit a ``probe-round-completed`` info line summarizing one sweep.
+
+    Info level — fires once per probe interval (default 60s). Dashboards
+    render this as "probes/min" and "failure ratio" without needing to
+    aggregate the individual ``probe-completed`` lines.
+    """
+    payload: ProbeRoundCompletedPayload = {
+        "providers_probed": providers_probed,
+        "failures": failures,
+    }
+    logger.info("probe-round-completed", extra=payload)
+
+
+class ProbeCapabilitiesDriftPayload(TypedDict):
+    """Structured shape of the ``probe-capabilities-drift`` log record.
+
+    Fields
+        provider: the provider whose probe response model mismatched.
+        configured_model: the model string in providers.yaml.
+        observed_model: the model name returned by the probe response.
+        in_registry: whether the observed model has an entry in the
+            capability registry.
+    """
+
+    provider: str
+    configured_model: str
+    observed_model: str
+    in_registry: bool
+
+
+def log_probe_capabilities_drift(
+    logger: logging.Logger,
+    *,
+    provider: str,
+    configured_model: str,
+    observed_model: str,
+    in_registry: bool,
+) -> None:
+    """Emit a ``probe-capabilities-drift`` warning line.
+
+    Fired when the continuous probe detects that the model name returned
+    by the provider differs from the configured model. This can indicate
+    a model swap (Ollama auto-updated), a misconfiguration, or a new
+    model that the capability registry doesn't know about yet.
+
+    Warn level — model drift can cause subtle behavior changes (e.g. a
+    model that supports thinking being replaced by one that doesn't).
+    """
+    payload: ProbeCapabilitiesDriftPayload = {
+        "provider": provider,
+        "configured_model": configured_model,
+        "observed_model": observed_model,
+        "in_registry": in_registry,
+    }
+    logger.warning("probe-capabilities-drift", extra=payload)

coderouter/metrics/collector.py

@@ -48,6 +48,13 @@ Event inventory (dispatch table in :meth:`MetricsCollector._dispatch`)
                                    + per-profile counter + latest_ratio gauge
     ``context-budget-trimmed`` (v2.0-F)→ ``context_budget_trims_total``
                                    + per-profile counter
+    ``drift-detected`` (v2.0-G)  → ``drift_detected_total`` + per-provider
+    ``drift-promoted`` (v2.0-G)  → ``drift_promoted_total``
+    ``drift-reload-attempted``   → ``drift_reload_total`` / success
+    ``partial-stitch-surfaced``  → ``partial_stitch_surfaced_total`` (v2.0-H)
+    ``probe-completed`` (v2.0-I) → ``probe_total`` / ``probe_success`` / ``probe_failure``
+                                   + per-provider latency gauge
+    ``probe-round-completed``    → ``probe_rounds_total`` (v2.0-I)
     ``coderouter-startup``       → ``startup_info`` (stored for the UI header)
 
     Unrecognized events are ignored (forward-compat: adding a new log
@@ -194,6 +201,29 @@ class MetricsCollector(logging.Handler):
         self._context_budget_trims_by_profile: Counter[str] = Counter()
         self._context_budget_latest_ratio: dict[str, float] = {}
 
+        # v2.0-G (L4): drift detection counters. Per-provider counts of
+        # drift events, promotions (rank demotions), and reload attempts.
+        self._drift_detected_total: int = 0
+        self._drift_detected_by_provider: Counter[str] = Counter()
+        self._drift_promoted_total: int = 0
+        self._drift_reload_total: int = 0
+        self._drift_reload_success_total: int = 0
+
+        # v2.0-H (L6): partial stitch surfaced counter. Tracks how often
+        # the mid-stream failure recovery delivered partial content to the
+        # client instead of a generic error event.
+        self._partial_stitch_surfaced_total: int = 0
+
+        # v2.0-I: continuous probe counters. Per-provider probe attempts
+        # and outcomes, plus a round counter for the dashboard's
+        # "probes/min" panel.
+        self._probe_total: Counter[str] = Counter()  # per-provider total probes
+        self._probe_success: Counter[str] = Counter()  # per-provider successes
+        self._probe_failure: Counter[str] = Counter()  # per-provider failures
+        self._probe_rounds_total: int = 0
+        self._probe_latency_ms: dict[str, float] = {}  # per-provider latest
+        self._probe_drift_detected: Counter[str] = Counter()  # per-provider drift
+
         # Last-error snapshot per provider (overwrites previous). Enables the
         # dashboard's "last error" column without scanning the ring.
         self._last_error: dict[str, dict[str, Any]] = {}
@@ -372,6 +402,43 @@ class MetricsCollector(logging.Handler):
                 profile = _str(extras.get("profile"))
                 self._context_budget_trims_total += 1
                 self._context_budget_trims_by_profile[profile] += 1
+            elif event == "drift-detected":
+                # v2.0-G (L4): drift detection fired.
+                provider = _str(extras.get("provider"))
+                self._drift_detected_total += 1
+                self._drift_detected_by_provider[provider] += 1
+                self._push_recent(event, extras, record)
+            elif event == "drift-promoted":
+                # v2.0-G (L4): drifted provider was demoted.
+                self._drift_promoted_total += 1
+                self._push_recent(event, extras, record)
+            elif event == "drift-reload-attempted":
+                # v2.0-G (L4): Ollama KV cache flush attempted.
+                self._drift_reload_total += 1
+                if extras.get("success"):
+                    self._drift_reload_success_total += 1
+            elif event == "partial-stitch-surfaced":
+                # v2.0-H (L6): mid-stream failure gracefully surfaced.
+                self._partial_stitch_surfaced_total += 1
+                self._push_recent(event, extras, record)
+            elif event == "probe-completed":
+                # v2.0-I: per-provider probe outcome.
+                provider = _str(extras.get("provider"))
+                self._probe_total[provider] += 1
+                if extras.get("success"):
+                    self._probe_success[provider] += 1
+                else:
+                    self._probe_failure[provider] += 1
+                latency_raw = extras.get("latency_ms")
+                if isinstance(latency_raw, int | float):
+                    self._probe_latency_ms[provider] = float(latency_raw)
+            elif event == "probe-round-completed":
+                # v2.0-I: round counter for the dashboard.
+                self._probe_rounds_total += 1
+            elif event == "probe-capabilities-drift":
+                # v2.0-I: model mismatch detected by probe.
+                provider = _str(extras.get("provider"))
+                self._probe_drift_detected[provider] += 1
             elif event == "coderouter-startup":
                 # Snapshot a subset — startup payload contains lists that are
                 # safe to surface to /metrics.json. Version / providers /
@@ -534,6 +601,23 @@ class MetricsCollector(logging.Handler):
                     "context_budget_latest_ratio": dict(
                         self._context_budget_latest_ratio
                     ),
+                    # v2.0-G (L4): drift detection aggregate counters.
+                    "drift_detected_total": self._drift_detected_total,
+                    "drift_detected_by_provider": dict(
+                        self._drift_detected_by_provider
+                    ),
+                    "drift_promoted_total": self._drift_promoted_total,
+                    "drift_reload_total": self._drift_reload_total,
+                    "drift_reload_success_total": self._drift_reload_success_total,
+                    # v2.0-H (L6): partial stitch surfaced.
+                    "partial_stitch_surfaced_total": self._partial_stitch_surfaced_total,
+                    # v2.0-I: continuous probe counters.
+                    "probe_total": dict(self._probe_total),
+                    "probe_success": dict(self._probe_success),
+                    "probe_failure": dict(self._probe_failure),
+                    "probe_rounds_total": self._probe_rounds_total,
+                    "probe_latency_ms": dict(self._probe_latency_ms),
+                    "probe_drift_detected": dict(self._probe_drift_detected),
                 },
                 "providers": provider_rows,
                 "recent": list(self._recent),
@@ -578,6 +662,15 @@ class MetricsCollector(logging.Handler):
             self._cost_savings_usd.clear()
             self._cost_total_usd_aggregate = 0.0
             self._cost_savings_usd_aggregate = 0.0
+            # v2.0-H (L6)
+            self._partial_stitch_surfaced_total = 0
+            # v2.0-I
+            self._probe_total.clear()
+            self._probe_success.clear()
+            self._probe_failure.clear()
+            self._probe_rounds_total = 0
+            self._probe_latency_ms.clear()
+            self._probe_drift_detected.clear()
             # v2.0-F (L1)
             self._context_budget_warnings_total = 0
             self._context_budget_trims_total = 0

coderouter/metrics/prometheus.py

@@ -381,6 +381,147 @@ def format_prometheus(snapshot: dict[str, Any]) -> str:
             samples=ratio_samples,
         )
     )
+
+    # ---- v2.0-G (L4): drift detection metrics ------------------------------
+    lines.extend(
+        _counter(
+            name="drift_detected_total",
+            help_text=(
+                "Drift detection events (quality degradation detected), "
+                "by provider."
+            ),
+            samples=[
+                ((("provider", p),), v)
+                for p, v in sorted(
+                    counters.get("drift_detected_by_provider", {}).items()
+                )
+            ],
+        )
+    )
+    drift_promoted = counters.get("drift_promoted_total", 0)
+    if drift_promoted:
+        lines.extend(
+            _counter(
+                name="drift_promoted_total",
+                help_text=(
+                    "Number of times a drifted provider was demoted "
+                    "(promote/reload action fired)."
+                ),
+                samples=[(((),), drift_promoted)],
+            )
+        )
+    drift_reload = counters.get("drift_reload_total", 0)
+    if drift_reload:
+        lines.extend(
+            _counter(
+                name="drift_reload_total",
+                help_text="Ollama KV cache flush attempts (reload action).",
+                samples=[(((),), drift_reload)],
+            )
+        )
+    drift_reload_ok = counters.get("drift_reload_success_total", 0)
+    if drift_reload_ok:
+        lines.extend(
+            _counter(
+                name="drift_reload_success_total",
+                help_text="Successful Ollama KV cache flush attempts.",
+                samples=[(((),), drift_reload_ok)],
+            )
+        )
+
+    # ---- v2.0-H (L6): partial stitch surfaced metric -----------------------
+    partial_stitch = counters.get("partial_stitch_surfaced_total", 0)
+    if partial_stitch:
+        lines.extend(
+            _counter(
+                name="partial_stitch_surfaced_total",
+                help_text=(
+                    "Mid-stream failures where partial content was delivered "
+                    "to the client (partial_stitch_action=surface)."
+                ),
+                samples=[((), partial_stitch)],
+            )
+        )
+
+    # ---- v2.0-I: continuous probe metrics ------------------------------------
+    probe_total_samples: list[tuple[tuple[tuple[str, str], ...], int]] = []
+    for provider, count in sorted(counters.get("probe_total", {}).items()):
+        probe_total_samples.append(
+            ((("provider", provider),), count)
+        )
+    if probe_total_samples:
+        lines.extend(
+            _counter(
+                name="probe_total",
+                help_text=(
+                    "Continuous health probe attempts, by provider. "
+                    "Each probe sends a 1-token completion to verify "
+                    "the full model pipeline."
+                ),
+                samples=probe_total_samples,
+            )
+        )
+    probe_outcome_samples: list[tuple[tuple[tuple[str, str], ...], int]] = []
+    for provider, count in sorted(counters.get("probe_success", {}).items()):
+        probe_outcome_samples.append(
+            ((("provider", provider), ("outcome", "success")), count)
+        )
+    for provider, count in sorted(counters.get("probe_failure", {}).items()):
+        probe_outcome_samples.append(
+            ((("provider", provider), ("outcome", "failure")), count)
+        )
+    if probe_outcome_samples:
+        lines.extend(
+            _counter(
+                name="probe_outcomes_total",
+                help_text=(
+                    "Continuous probe outcomes by provider and result "
+                    "(success | failure)."
+                ),
+                samples=probe_outcome_samples,
+            )
+        )
+    probe_rounds = counters.get("probe_rounds_total", 0)
+    if probe_rounds:
+        lines.extend(
+            _counter(
+                name="probe_rounds_total",
+                help_text="Completed probe sweep rounds (one round probes all eligible providers).",
+                samples=[((), probe_rounds)],
+            )
+        )
+    probe_drift_samples: list[tuple[tuple[tuple[str, str], ...], int]] = [
+        ((("provider", p),), v)
+        for p, v in sorted(counters.get("probe_drift_detected", {}).items())
+    ]
+    if probe_drift_samples:
+        lines.extend(
+            _counter(
+                name="probe_drift_detected_total",
+                help_text=(
+                    "Model-name mismatches detected by continuous probing "
+                    "(configured model != response model), by provider."
+                ),
+                samples=probe_drift_samples,
+            )
+        )
+    # Gauge: latest probe latency per provider (ms).
+    latency_samples: list[tuple[tuple[tuple[str, str], ...], float]] = [
+        ((("provider", p),), round(v, 1))
+        for p, v in sorted(counters.get("probe_latency_ms", {}).items())
+    ]
+    if latency_samples:
+        lines.extend(
+            _gauge_float(
+                name="probe_latency_ms",
+                help_text=(
+                    "Latest probe round-trip latency in milliseconds, by "
+                    "provider. Gauge (most recent value, not cumulative)."
+                ),
+                samples=latency_samples,
+            )
+        )
+
     return "\n".join(lines) + "\n"
 
 

coderouter/routing/adaptive.py

@@ -234,6 +234,29 @@ class AdaptiveAdjuster:
         while entry.observations and entry.observations[0].ts_monotonic < cutoff:
             entry.observations.popleft()
 
+    def demote(self, provider: str, *, steps: int = 2) -> None:
+        """Force-demote a provider by injecting synthetic failure observations.
+
+        Used by v2.0-G drift detection to push a provider's error rate above
+        the demotion threshold (``ERROR_RATE_DEMOTE_THRESHOLD``). Each step
+        injects one synthetic failure observation, so ``steps=2`` guarantees
+        the provider will be ranked lower on the next ``compute_effective_order``.
+
+        The injected observations carry no latency signal (``latency_ms=None``)
+        and expire naturally after ``ROLLING_WINDOW_S`` seconds.
+        """
+        ts = time.monotonic()
+        with self._lock:
+            entry = self._state.setdefault(provider, _AdjusterState())
+            for _ in range(steps):
+                entry.observations.append(
+                    _ProviderObservation(
+                        ts_monotonic=ts,
+                        latency_ms=None,
+                        success=False,
+                    )
+                )
+
     # ------------------------------------------------------------------
     # Stats
     # ------------------------------------------------------------------