coderouter-cli 2.1.0__py3-none-any.whl → 2.2.0__py3-none-any.whl

coderouter/logging.py

@@ -595,6 +595,114 @@ def log_demote_unhealthy_provider(
     logger.info("demote-unhealthy-provider", extra=payload)
 
 
+# ---------------------------------------------------------------------------
+# v2.0-J: self-healing log shapes
+# ---------------------------------------------------------------------------
+
+
+class SelfHealingExcludePayload(TypedDict):
+    """Structured shape of the ``self-healing-exclude`` log record."""
+
+    provider: str
+    profile: str
+    consecutive_failures: int
+
+
+class SelfHealingRestorePayload(TypedDict):
+    """Structured shape of the ``self-healing-restore`` log record."""
+
+    provider: str
+    profile: str
+    excluded_duration_s: float
+
+
+class SelfHealingRestartPayload(TypedDict):
+    """Structured shape of the ``self-healing-restart`` log record."""
+
+    provider: str
+    command: str
+    success: bool
+    error: str | None
+
+
+class SelfHealingRecoveryProbePayload(TypedDict):
+    """Structured shape of the ``self-healing-recovery-probe`` log record."""
+
+    provider: str
+    success: bool
+    next_interval_s: float
+    latency_ms: float
+
+
+def log_self_healing_exclude(
+    logger: logging.Logger,
+    *,
+    provider: str,
+    profile: str,
+    consecutive_failures: int,
+) -> None:
+    """Emit when a provider is excluded from the chain by self-healing."""
+    payload: SelfHealingExcludePayload = {
+        "provider": provider,
+        "profile": profile,
+        "consecutive_failures": consecutive_failures,
+    }
+    logger.warning("self-healing-exclude", extra=payload)
+
+
+def log_self_healing_restore(
+    logger: logging.Logger,
+    *,
+    provider: str,
+    profile: str,
+    excluded_duration_s: float,
+) -> None:
+    """Emit when a previously excluded provider is restored to the chain."""
+    payload: SelfHealingRestorePayload = {
+        "provider": provider,
+        "profile": profile,
+        "excluded_duration_s": round(excluded_duration_s, 1),
+    }
+    logger.info("self-healing-restore", extra=payload)
+
+
+def log_self_healing_restart(
+    logger: logging.Logger,
+    *,
+    provider: str,
+    command: str,
+    success: bool,
+    error: str | None = None,
+) -> None:
+    """Emit after attempting to restart a provider's backend process."""
+    payload: SelfHealingRestartPayload = {
+        "provider": provider,
+        "command": command,
+        "success": success,
+        "error": error,
+    }
+    level = logging.INFO if success else logging.WARNING
+    logger.log(level, "self-healing-restart", extra=payload)
+
+
+def log_self_healing_recovery_probe(
+    logger: logging.Logger,
+    *,
+    provider: str,
+    success: bool,
+    next_interval_s: float,
+    latency_ms: float,
+) -> None:
+    """Emit after each recovery probe attempt for an excluded provider."""
+    payload: SelfHealingRecoveryProbePayload = {
+        "provider": provider,
+        "success": success,
+        "next_interval_s": round(next_interval_s, 1),
+        "latency_ms": round(latency_ms, 1),
+    }
+    logger.info("self-healing-recovery-probe", extra=payload)
+
+
 # ---------------------------------------------------------------------------
 # v1.0-A: output-filter-applied log shape
 #

coderouter/metrics/collector.py

@@ -623,6 +623,81 @@ class MetricsCollector(logging.Handler):
                 "recent": list(self._recent),
             }
 
+    # ------------------------------------------------------------------
+    # v2.0-K: Persistence
+    # ------------------------------------------------------------------
+
+    def save_state(self) -> dict[str, object]:
+        """Export key counters for cross-restart persistence.
+
+        Returns a JSON-safe dict of the most operationally-important
+        counters.  The ``recent`` ring and per-provider ``last_error``
+        are excluded (ephemeral by nature).
+        """
+        with self._lock:
+            return {
+                "requests_total": self._requests_total,
+                "provider_attempts": dict(self._provider_attempts),
+                "provider_outcomes": {
+                    k: dict(v) for k, v in self._provider_outcomes.items()
+                },
+                "cost_total_usd": dict(self._cost_total_usd),
+                "cost_savings_usd": dict(self._cost_savings_usd),
+                "cost_total_usd_aggregate": self._cost_total_usd_aggregate,
+                "cost_savings_usd_aggregate": self._cost_savings_usd_aggregate,
+                "chain_paid_gate_blocked_total": self._chain_paid_gate_blocked_total,
+                "chain_budget_exceeded_total": self._chain_budget_exceeded_total,
+                "chain_memory_pressure_blocked_total": self._chain_memory_pressure_blocked_total,
+                "chain_uniform_auth_failure_total": self._chain_uniform_auth_failure_total,
+                "probe_rounds_total": self._probe_rounds_total,
+            }
+
+    def load_state(self, state: dict[str, object]) -> None:
+        """Restore counters from a previously saved dict.
+
+        Additive: values from ``state`` are *added* to the current
+        (zeroed) counters, so calling ``load_state`` on a fresh
+        collector restores the prior session's totals.
+        """
+        if not isinstance(state, dict):
+            return
+        with self._lock:
+            self._requests_total += int(state.get("requests_total", 0))
+            for k, v in (state.get("provider_attempts") or {}).items():
+                self._provider_attempts[k] += int(v)
+            for prov, outcomes in (state.get("provider_outcomes") or {}).items():
+                if not isinstance(outcomes, dict):
+                    continue
+                if prov not in self._provider_outcomes:
+                    self._provider_outcomes[prov] = Counter()
+                for k, v in outcomes.items():
+                    self._provider_outcomes[prov][k] += int(v)
+            for k, v in (state.get("cost_total_usd") or {}).items():
+                self._cost_total_usd[k] = self._cost_total_usd.get(k, 0.0) + float(v)
+            for k, v in (state.get("cost_savings_usd") or {}).items():
+                self._cost_savings_usd[k] = self._cost_savings_usd.get(k, 0.0) + float(v)
+            self._cost_total_usd_aggregate += float(
+                state.get("cost_total_usd_aggregate", 0.0)
+            )
+            self._cost_savings_usd_aggregate += float(
+                state.get("cost_savings_usd_aggregate", 0.0)
+            )
+            self._chain_paid_gate_blocked_total += int(
+                state.get("chain_paid_gate_blocked_total", 0)
+            )
+            self._chain_budget_exceeded_total += int(
+                state.get("chain_budget_exceeded_total", 0)
+            )
+            self._chain_memory_pressure_blocked_total += int(
+                state.get("chain_memory_pressure_blocked_total", 0)
+            )
+            self._chain_uniform_auth_failure_total += int(
+                state.get("chain_uniform_auth_failure_total", 0)
+            )
+            self._probe_rounds_total += int(
+                state.get("probe_rounds_total", 0)
+            )
+
     # ------------------------------------------------------------------
     # Test hook
     # ------------------------------------------------------------------

coderouter/output_filters.py

@@ -52,6 +52,7 @@ __all__ = [
     "OutputFilterChain",
     "StripStopMarkersFilter",
     "StripThinkingFilter",
+    "StripToolCallXmlFilter",
     "apply_output_filters",
     "validate_output_filters",
 ]
@@ -63,20 +64,28 @@ __all__ = [
 
 
 DEFAULT_STOP_MARKERS: tuple[str, ...] = (
+    # v1.0-A originals
     "<|turn|>",
     "<|end|>",
     "<|python_tag|>",
     "<|im_end|>",
     "<|eot_id|>",
     "<|channel>thought",
+    # v2.2: tool-call XML tags leaked by Qwen / Hermes / Llama tool-call
+    # formats. These appear when the model writes tool calls as XML
+    # instead of structured JSON, or when the tokenizer's special-token
+    # handling leaks through.
+    "<|tool▁call|>",
+    "<|tool▁sep|>",
 )
 """Default stop/harness markers stripped by ``strip_stop_markers``.
 
 Covers Llama 3.x (``<|python_tag|>``, ``<|eot_id|>``), ChatML / Qwen
-(``<|im_end|>``, ``<|end|>``), Gemma-ish (``<|turn|>``) and OpenAI-
-harmony (``<|channel>thought``). Extending this tuple is an ABI change
-— users who need a bespoke set can add a dedicated filter entry in
-a later minor; for v1.0-A the fixed list covers observed leaks.
+(``<|im_end|>``, ``<|end|>``), Gemma-ish (``<|turn|>``), OpenAI-
+harmony (``<|channel>thought``), and Qwen / Hermes tool-call markers
+(``<|tool▁call|>``, ``<|tool▁sep|>``). Extending this tuple is an ABI
+change — users who need a bespoke set can add a dedicated filter entry
+in a later minor.
 """
 
 
@@ -292,6 +301,87 @@ class StripStopMarkersFilter:
         return "".join(out_parts)
 
 
+# ---------------------------------------------------------------------------
+# strip_tool_call_xml (v2.2)
+# ---------------------------------------------------------------------------
+
+
+_TOOL_CALL_OPEN = "<tool_call>"
+_TOOL_CALL_CLOSE = "</tool_call>"
+
+
+class StripToolCallXmlFilter:
+    """Remove ``<tool_call>...</tool_call>`` XML blocks from assistant content.
+
+    Qwen / Hermes / Llama tool-call formats sometimes emit tool calls
+    as ``<tool_call>{"name": "Bash", ...}</tool_call>`` XML in the
+    content stream. When ``tool_repair`` has already extracted the
+    structured JSON from these blocks, the XML wrapper tags are
+    leftover debris that confuse downstream clients.
+
+    Architecture note: this filter should run AFTER ``tool_repair``
+    has had a chance to extract the JSON. The filter chain is applied
+    at the adapter boundary (post-repair), so ordering is naturally
+    correct.
+
+    Implementation mirrors ``StripThinkingFilter`` — the same
+    stateful open/close tag scanning, same chunk-boundary safety.
+    """
+
+    name = "strip_tool_call_xml"
+
+    def __init__(self) -> None:
+        """Initialize the per-request buffer + in-block state to empty."""
+        self.modified: bool = False
+        self._in_block: bool = False
+        self._buffer: str = ""
+
+    def feed(self, text: str, *, eof: bool = False) -> str:
+        """Consume ``text`` and return the portion safe to emit now.
+
+        Mirrors the ``StripThinkingFilter`` algorithm: greedy tag
+        matching with partial-prefix holdback across chunk boundaries.
+        """
+        self._buffer += text
+        out_parts: list[str] = []
+
+        while True:
+            if not self._in_block:
+                idx = self._buffer.find(_TOOL_CALL_OPEN)
+                if idx != -1:
+                    out_parts.append(self._buffer[:idx])
+                    self._buffer = self._buffer[idx + len(_TOOL_CALL_OPEN) :]
+                    self._in_block = True
+                    self.modified = True
+                    continue
+                # No open tag — emit all but a potential partial prefix.
+                overlap = _max_suffix_overlap(self._buffer, _TOOL_CALL_OPEN)
+                if overlap:
+                    out_parts.append(self._buffer[:-overlap])
+                    self._buffer = self._buffer[-overlap:]
+                else:
+                    out_parts.append(self._buffer)
+                    self._buffer = ""
+                break
+            # in_block: suppress until we find the close tag.
+            idx = self._buffer.find(_TOOL_CALL_CLOSE)
+            if idx != -1:
+                self._buffer = self._buffer[idx + len(_TOOL_CALL_CLOSE) :]
+                self._in_block = False
+                continue
+            # No close tag — retain potential partial suffix, drop the rest.
+            overlap = _max_suffix_overlap(self._buffer, _TOOL_CALL_CLOSE)
+            self._buffer = self._buffer[-overlap:] if overlap else ""
+            break
+
+        if eof:
+            if not self._in_block:
+                out_parts.append(self._buffer)
+            # If still in block at eof, silently drop the partial block.
+            self._buffer = ""
+        return "".join(out_parts)
+
+
 # ---------------------------------------------------------------------------
 # Registry + chain
 # ---------------------------------------------------------------------------
@@ -300,6 +390,7 @@ class StripStopMarkersFilter:
 KNOWN_FILTERS: dict[str, type[OutputFilter]] = {
     StripThinkingFilter.name: StripThinkingFilter,
     StripStopMarkersFilter.name: StripStopMarkersFilter,
+    StripToolCallXmlFilter.name: StripToolCallXmlFilter,
 }
 """Registry of string-name → filter class.
 

coderouter/routing/budget.py

@@ -187,5 +187,40 @@ class BudgetTracker:
             self._totals.clear()
             self._month = current
 
+    # ------------------------------------------------------------------
+    # v2.0-K: Persistence
+    # ------------------------------------------------------------------
+
+    def save_state(self) -> dict[str, object]:
+        """Export the current state as a JSON-safe dict.
+
+        Called by the engine to persist budget totals across restarts.
+        """
+        with self._lock:
+            return {
+                "month": self._month,
+                "totals": dict(self._totals),
+            }
+
+    def load_state(self, state: dict[str, object]) -> None:
+        """Restore state from a previously saved dict.
+
+        Only restores if the saved month matches the current month
+        (no point restoring last month's totals into a new month).
+        """
+        if not isinstance(state, dict):
+            return
+        saved_month = state.get("month", "")
+        with self._lock:
+            current = _utc_month_key()
+            if saved_month != current:
+                return  # stale month — skip
+            totals = state.get("totals", {})
+            if isinstance(totals, dict):
+                self._totals = {
+                    k: float(v) for k, v in totals.items() if isinstance(v, (int, float))
+                }
+                self._month = current
+
 
 __all__ = ["BudgetTracker"]

coderouter/routing/fallback.py

@@ -24,12 +24,16 @@ Dual entry points (v0.3.x-1):
 
 from __future__ import annotations
 
+import asyncio
 import time
 from collections.abc import AsyncIterator
 from typing import TYPE_CHECKING, Any, Final
 
 if TYPE_CHECKING:
+    from coderouter.config.schemas import FallbackChain
     from coderouter.guards.drift_detection import DriftVerdict
+    from coderouter.guards.self_healing import SelfHealingOrchestrator
+    from coderouter.state.store import StateStore
 
 from coderouter.adapters.anthropic_native import AnthropicAdapter
 from coderouter.adapters.base import (
@@ -51,7 +55,9 @@ from coderouter.guards.memory_pressure import (
 )
 from coderouter.guards.tool_loop import (
     DEFAULT_LOOP_INJECT_HINT,
+    ToolCountExceededError,
     ToolLoopBreakError,
+    check_total_tool_count,
     detect_tool_loop,
     inject_loop_break_hint,
 )
@@ -130,7 +136,8 @@ def _apply_tool_loop_guard(
 
     Returns the (possibly mutated) request. Raises
     :class:`ToolLoopBreakError` when the configured action is ``break``
-    and a loop was detected.
+    and a loop was detected. Also raises :class:`ToolCountExceededError`
+    when the total tool-call count exceeds ``max_tool_calls`` (v2.2).
 
     Profile resolution: uses ``request.profile`` (the X-CodeRouter-Mode
     header / explicit body field) and falls back to
@@ -149,6 +156,30 @@ def _apply_tool_loop_guard(
         # resolution path produces its own diagnostic.
         return request
 
+    # v2.2: total tool-call count hard cap — runs before streak
+    # detection because it's a cheaper O(n) scan that catches a
+    # broader class of runaway behavior.
+    if profile.max_tool_calls > 0:
+        exceeded = check_total_tool_count(
+            request,
+            max_calls=profile.max_tool_calls,
+        )
+        if exceeded is not None:
+            logger.warning(
+                "tool-count-exceeded",
+                extra={
+                    "profile": profile.name,
+                    "total_count": exceeded.total_count,
+                    "max_allowed": exceeded.max_allowed,
+                    "action": profile.tool_loop_action,
+                },
+            )
+            if profile.tool_loop_action == "break":
+                raise ToolCountExceededError(exceeded, profile.name)
+            # For "warn" and "inject" actions, log only and continue.
+            # The inject action's hint is not meaningful for count
+            # exceeded (not a same-tool loop), so we just warn.
+
     detection = detect_tool_loop(
         request,
         window=profile.tool_loop_window,
@@ -818,6 +849,12 @@ class FallbackEngine:
         # Distinct from v1.9-C ``adaptive`` which handles the
         # gradient case via a rolling window.
         self._backend_health_monitor: BackendHealthMonitor = BackendHealthMonitor()
+        # v2.0-J: self-healing orchestrator. Manages provider exclusion,
+        # restart, and recovery probing when backend_health_action is
+        # "exclude". Composes with the L5 backend health monitor.
+        from coderouter.guards.self_healing import SelfHealingOrchestrator
+
+        self._self_healing: SelfHealingOrchestrator = SelfHealingOrchestrator()
         # v2.0-G (L4): per-process drift detection window manager.
         # Stores per-provider rolling observations; the detector is
         # invoked after each provider-ok / provider-failed event and
@@ -831,6 +868,12 @@ class FallbackEngine:
         self._drift_demoted: dict[str, float] = {}
         # Last drift verdict (set by _observe_drift_signal for ingress header).
         self._last_drift_verdict: DriftVerdict | None = None
+        # v2.0-J: active recovery probe tasks (one per excluded provider).
+        self._recovery_tasks: dict[str, asyncio.Task[None]] = {}
+        # v2.0-J: shutdown event shared with recovery probe tasks.
+        self._recovery_shutdown: asyncio.Event | None = None
+        # v2.0-K: persistent state store (None = in-memory only).
+        self._state_store: StateStore | None = None
 
     @property
     def last_drift_severity(self) -> str | None:
@@ -914,6 +957,20 @@ class FallbackEngine:
     def _backend_health(self) -> BackendHealthMonitor:
         return self.backend_health
 
+    @property
+    def self_healing(self) -> SelfHealingOrchestrator:
+        """Return the v2.0-J self-healing orchestrator.
+
+        Lazy init for backward compat with __new__-constructed test engines.
+        """
+        from coderouter.guards.self_healing import SelfHealingOrchestrator
+
+        existing = getattr(self, "_self_healing", None)
+        if existing is None:
+            self._self_healing = SelfHealingOrchestrator()
+            existing = self._self_healing
+        return existing
+
     def _observe_provider_failure(
         self,
         provider: str,
@@ -991,6 +1048,18 @@ class FallbackEngine:
                     new_state=transition.new_state,
                     consecutive_failures=transition.consecutive_failures,
                 )
+                # v2.0-J: trigger self-healing on UNHEALTHY + exclude.
+                if (
+                    transition.new_state == "UNHEALTHY"
+                    and bh_action == "exclude"
+                ):
+                    newly_excluded = self.self_healing.on_unhealthy(
+                        provider,
+                        profile=chosen,
+                        consecutive_failures=transition.consecutive_failures,
+                    )
+                    if newly_excluded:
+                        self._spawn_recovery_probe(provider, chain=chain)
 
     def _observe_provider_success(
         self,
@@ -1032,6 +1101,134 @@ class FallbackEngine:
                 consecutive_failures=transition.consecutive_failures,
             )
 
+    def _spawn_recovery_probe(
+        self,
+        provider: str,
+        *,
+        chain: FallbackChain,
+    ) -> None:
+        """Launch an async recovery probe task for an excluded provider.
+
+        v2.0-J: called by ``_observe_provider_failure`` when a provider
+        is newly excluded. The task runs ``recovery_probe_loop`` with
+        exponential backoff until the provider recovers or shutdown.
+
+        Safe to call from a sync context — uses ``asyncio.get_event_loop``
+        to schedule the task. No-op if no running event loop (e.g. in
+        pure-sync tests).
+        """
+        import asyncio
+
+        from coderouter.guards.self_healing import recovery_probe_loop
+
+        # Find the ProviderConfig for this provider name.
+        provider_config = None
+        for p in self.config.providers:
+            if p.name == provider:
+                provider_config = p
+                break
+        if provider_config is None:
+            return
+
+        # Reuse or create a shared shutdown event.
+        if self._recovery_shutdown is None:
+            self._recovery_shutdown = asyncio.Event()
+
+        # Don't spawn duplicate tasks.
+        existing = self._recovery_tasks.get(provider)
+        if existing is not None and not existing.done():
+            return
+
+        try:
+            loop = asyncio.get_running_loop()
+        except RuntimeError:
+            return  # no event loop — skip (sync test context)
+
+        task = loop.create_task(
+            recovery_probe_loop(
+                provider_config,
+                orchestrator=self.self_healing,
+                record_fn=self.backend_health.record_attempt,
+                health_threshold=chain.backend_health_threshold,
+                initial_interval_s=chain.recovery_probe_initial_s,
+                max_interval_s=chain.recovery_probe_max_s,
+                restart_timeout_s=chain.restart_timeout_s,
+                probe_timeout_s=10.0,
+                shutdown_event=self._recovery_shutdown,
+                profile=chain.name,
+            ),
+            name=f"recovery-probe-{provider}",
+        )
+        self._recovery_tasks[provider] = task
+
+    async def shutdown_recovery_probes(self) -> None:
+        """Signal all recovery probe tasks to stop and await them.
+
+        Called from the app lifespan shutdown path.
+        """
+        import contextlib
+
+        if self._recovery_shutdown is not None:
+            self._recovery_shutdown.set()
+        for task in self._recovery_tasks.values():
+            if not task.done():
+                with contextlib.suppress(Exception):
+                    await task
+        self._recovery_tasks.clear()
+
+    # ------------------------------------------------------------------
+    # v2.0-K: State persistence
+    # ------------------------------------------------------------------
+
+    def attach_state_store(self, store: StateStore) -> None:
+        """Attach a :class:`StateStore` and load persisted state.
+
+        Called from the app lifespan startup path when ``state_dir``
+        is configured.  Loads budget, health, self-healing, and
+        metrics state from the store.
+        """
+        self._state_store = store
+        self._load_all_state()
+
+    def save_all_state(self) -> None:
+        """Persist all subsystem state to the attached store.
+
+        Called from the app lifespan shutdown path and optionally
+        on a periodic timer.  No-op if no store is attached.
+        """
+        store = self._state_store
+        if store is None:
+            return
+        import contextlib
+
+        with contextlib.suppress(Exception):
+            store.put("budget", "state", self._budget.save_state())
+        with contextlib.suppress(Exception):
+            store.put("health", "state", self.backend_health.save_state())
+        with contextlib.suppress(Exception):
+            store.put("self_healing", "state", self.self_healing.save_state())
+        # MetricsCollector state is saved separately via the singleton.
+
+    def _load_all_state(self) -> None:
+        """Restore subsystem state from the attached store."""
+        store = self._state_store
+        if store is None:
+            return
+        import contextlib
+
+        with contextlib.suppress(Exception):
+            budget_state = store.get("budget", "state")
+            if budget_state is not None:
+                self._budget.load_state(budget_state)  # type: ignore[arg-type]
+        with contextlib.suppress(Exception):
+            health_state = store.get("health", "state")
+            if health_state is not None:
+                self.backend_health.load_state(health_state)  # type: ignore[arg-type]
+        with contextlib.suppress(Exception):
+            sh_state = store.get("self_healing", "state")
+            if sh_state is not None:
+                self.self_healing.load_state(sh_state)  # type: ignore[arg-type]
+
     def _observe_drift_signal(
         self,
         provider: str,
@@ -1340,6 +1537,19 @@ class FallbackEngine:
                         profile=chosen,
                     )
                 adapters = healthy + unhealthy
+
+        # Pass 4b: v2.0-J self-healing exclusion. When the action is
+        # "exclude", providers in the orchestrator's excluded set are
+        # removed entirely from the chain. Unlike "demote" (which
+        # moves to the back), excluded providers are not attempted at
+        # all — recovery probes run in the background to detect when
+        # they come back. If all providers are excluded, fall through
+        # to the existing NoProvidersAvailableError path.
+        if chain.backend_health_action == "exclude":
+            excluded = self.self_healing.excluded_providers()
+            if excluded:
+                adapters = [a for a in adapters if a.name not in excluded]
+
         return adapters
 
     def _resolve_anthropic_chain(self, request: AnthropicRequest) -> list[tuple[BaseAdapter, bool]]:

coderouter/state/__init__.py

@@ -0,0 +1,15 @@
+"""Persistent state layer (v2.0-K).
+
+Four modules:
+
+* :mod:`coderouter.state.store`       — sqlite3 KV store for operational
+                                         metadata (budget totals, health
+                                         state, self-healing exclusions).
+* :mod:`coderouter.state.audit_log`   — JSONL structured event log with
+                                         rotation and CLI reader.
+* :mod:`coderouter.state.request_log` — JSONL request metadata journal
+                                         (per-request token counts, cost,
+                                         provider — no request body).
+* :mod:`coderouter.state.replay`      — Statistical A/B analysis engine
+                                         over request journal entries.
+"""