coderouter-cli 1.9.0__py3-none-any.whl → 1.10.0__py3-none-any.whl

coderouter/config/schemas.py

@@ -128,6 +128,27 @@ class CostConfig(BaseModel):
             "(unusual but theoretically supported by the schema)."
         ),
     )
+    monthly_budget_usd: float | None = Field(
+        default=None,
+        ge=0.0,
+        description=(
+            "v1.10 (LiteLLM 由来 / v1.9-D の累積版): per-provider "
+            "monthly USD spend cap. When set, the engine's chain "
+            "resolver skips this provider and emits "
+            "``skip-budget-exceeded`` once the running per-provider "
+            "total for the current calendar month (UTC) reaches or "
+            "exceeds this value. Unset (None) = no cap (default). "
+            "\n\n"
+            "Reset semantics: in-memory only — running totals zero "
+            "out on process restart and on UTC calendar-month "
+            "rollover. Operators who need durable budget state "
+            "across restarts should pair this with external "
+            "monitoring on the cost dashboard's ``cost_total_usd`` "
+            "panel; persistent budget state is out of scope for "
+            "v1.10 (no on-disk store, no Redis, etc., per the "
+            "5-deps invariant in plan.md §5.4)."
+        ),
+    )
 
 
 class ProviderConfig(BaseModel):
@@ -317,6 +338,95 @@ class FallbackChain(BaseModel):
             "error response. See FallbackChain comment for trade-offs."
         ),
     )
+    # v1.9-E phase 2 (L2): memory-pressure detection + cooldown.
+    #
+    # Local backends (Ollama / LM Studio / llama.cpp) report VRAM
+    # exhaustion via 5xx responses with bodies like "out of memory" /
+    # "CUDA out of memory" / "insufficient memory". When the chain
+    # encounters one of these, marking the provider as "pressured"
+    # for a cooldown window prevents the engine from re-hammering the
+    # same exhausted backend on the very next request — the chain
+    # falls through to the next provider, which is typically a
+    # lighter-weight model or a remote fallback that has the headroom.
+    #
+    # Three actions trade off intervention against operator preference:
+    #   * ``off``   — no detection / no logging / no skip. Backward-compat default.
+    #   * ``warn``  — emit ``memory-pressure-detected`` log when an OOM
+    #                 error is observed; do not skip on subsequent calls.
+    #   * ``skip``  — ``warn`` + put the provider in a cooldown window;
+    #                 subsequent chain resolves filter it out and emit
+    #                 ``skip-memory-pressure`` until the cooldown expires.
+    memory_pressure_action: Literal["off", "warn", "skip"] = Field(
+        default="warn",
+        description=(
+            "v1.9-E (L2 phase 2): action on observed backend OOM "
+            "(provider failure with an out-of-memory error body). "
+            "``warn`` (default) logs only — diagnostic, no chain "
+            "behavior change. ``skip`` enters a cooldown window so "
+            "the next request's chain resolver filters the pressured "
+            "provider out and falls through to the next entry. "
+            "``off`` disables the detector entirely (zero "
+            "observation overhead, identical to v1.9.x behavior)."
+        ),
+    )
+    memory_pressure_cooldown_s: int = Field(
+        default=120,
+        ge=10,
+        le=3600,
+        description=(
+            "v1.9-E (L2 phase 2): cooldown window in seconds applied "
+            "after an OOM detection when ``memory_pressure_action`` "
+            "is ``skip``. Default 120 s gives the local backend "
+            "enough time to release model state from VRAM before the "
+            "engine re-attempts. Capped at 3600 s (1 hour) — anything "
+            "longer is better expressed as marking the provider "
+            "``paid: true`` and bouncing the process."
+        ),
+    )
+    # v1.9-E phase 2 (L5): backend health monitoring (passive).
+    #
+    # A consecutive-failure state machine per provider:
+    #   * HEALTHY   — no recent failures (initial state).
+    #   * DEGRADED  — ``backend_health_threshold`` consecutive failures
+    #                 observed; the provider has lost its "fresh" status
+    #                 but is still attempted in chain order.
+    #   * UNHEALTHY — ``2 x backend_health_threshold`` consecutive
+    #                 failures; depending on the action, the provider
+    #                 is either demoted to chain end or skipped entirely.
+    # A single success on ``provider-ok`` resets the counter and the
+    # state to HEALTHY immediately — no rolling window, no debounce.
+    # Distinct from the v1.9-C ``adaptive`` gradient (continuous
+    # latency / error-rate buffer with debounce) which handles the
+    # "slow but alive" case; L5 handles the "hard crash" case.
+    backend_health_action: Literal["off", "warn", "demote"] = Field(
+        default="warn",
+        description=(
+            "v1.9-E (L5 phase 2): action when a provider transitions "
+            "to UNHEALTHY (consecutive failures crossed the threshold). "
+            "``warn`` (default) emits a state-change log line only — "
+            "diagnostic, no chain reorder. ``demote`` additionally "
+            "moves the UNHEALTHY provider to the back of the chain "
+            "for the next ``_resolve_chain`` (similar to v1.9-C "
+            "adaptive demotion but state-machine-based, not "
+            "rolling-window-based). ``off`` disables the monitor "
+            "entirely (zero observation overhead, identical to "
+            "v1.9.x behavior)."
+        ),
+    )
+    backend_health_threshold: int = Field(
+        default=3,
+        ge=2,
+        le=20,
+        description=(
+            "v1.9-E (L5 phase 2): consecutive-failure count that "
+            "triggers the HEALTHY → DEGRADED transition. The "
+            "DEGRADED → UNHEALTHY transition fires at ``2x`` this "
+            "value. Default 3 catches "
+            "Ollama / LM Studio crashes (which produce a deterministic "
+            "5xx pattern on every retry) without flapping on transient "
+            "blips that the v1.9-C adaptive adjuster already handles."
+        ),
+    )
     adaptive: bool = Field(
         default=False,
         description=(
@@ -358,6 +468,36 @@ class RuleMatcher(BaseModel):
     - ``content_contains: "foo"`` — substring match (case-sensitive).
     - ``content_regex: r"..."`` — Python ``re.search``; compiled at
       model-construction time so typos fail startup.
+
+    Variants ([Unreleased] / per-model auto-routing, free-claude-code 由来):
+
+    - ``model_pattern: r"claude-3-5-haiku.*"`` — Python ``re.fullmatch``
+      against the request body's ``model`` field. Lets clients route on
+      the model identifier the agent (Claude Code / Cursor) sent
+      (Opus / Sonnet / Haiku → different profiles) without needing an
+      explicit ``profile`` field on the wire. Compiled at load like
+      ``content_regex``. ``fullmatch`` semantics (vs ``search`` for
+      ``content_regex``) because model identifiers are structured tokens
+      — users typically describe the whole identifier with a wildcard
+      tail, not an arbitrary substring.
+
+    Variants ([Unreleased] / longContext auto-switch, claude-code-router
+    由来):
+
+    - ``content_token_count_min: 32000`` — char-count ÷ 4 heuristic
+      across **all** messages in the request body (not just the
+      latest user message — this matcher describes the request's
+      overall size). When the estimated token count is ``>=`` the
+      threshold, route to a long-context profile (typically pointing
+      at Gemini Flash 1M ctx, Haiku 200K, etc.). Distinct from the
+      other content matchers which operate on the latest user
+      message only — context-window pressure is a request-shape
+      property, not a per-turn property. The estimator deliberately
+      avoids tiktoken / SentencePiece (forbidden by the 5-deps
+      invariant in plan.md §5.4); operators with non-English-heavy
+      workloads can compensate by tuning the threshold, since the
+      char/4 heuristic is conservative for CJK and looser for
+      English code.
     """
 
     model_config = ConfigDict(extra="forbid")
@@ -366,12 +506,16 @@ class RuleMatcher(BaseModel):
     code_fence_ratio_min: float | None = Field(default=None, ge=0.0, le=1.0)
     content_contains: str | None = None
     content_regex: str | None = None
+    model_pattern: str | None = None
+    content_token_count_min: int | None = Field(default=None, ge=1)
 
     _MATCHER_FIELDS: tuple[str, ...] = (
         "has_image",
         "code_fence_ratio_min",
         "content_contains",
         "content_regex",
+        "model_pattern",
+        "content_token_count_min",
     )
 
     @model_validator(mode="after")
@@ -388,7 +532,9 @@ class RuleMatcher(BaseModel):
 
     @model_validator(mode="after")
     def _compile_regex_eagerly(self) -> Self:
-        """Compile ``content_regex`` at load so bad patterns fail startup."""
+        """Compile ``content_regex`` / ``model_pattern`` at load so bad
+        patterns fail startup rather than at first request.
+        """
         if self.content_regex is not None:
             try:
                 re.compile(self.content_regex)
@@ -396,6 +542,13 @@ class RuleMatcher(BaseModel):
                 raise ValueError(
                     f"Invalid regex for content_regex {self.content_regex!r}: {exc}"
                 ) from exc
+        if self.model_pattern is not None:
+            try:
+                re.compile(self.model_pattern)
+            except re.error as exc:
+                raise ValueError(
+                    f"Invalid regex for model_pattern {self.model_pattern!r}: {exc}"
+                ) from exc
         return self
 
 

coderouter/doctor.py

@@ -448,7 +448,7 @@ _STREAMING_PROBE_MIN_EXPECTED_CHARS = 40
 # trace plus the actual answer.
 #
 # Numbers picked from the v1.8.1 reality-check session
-# (docs/articles/note-v1-8-1-reality-check.md):
+# (docs/articles/v1-saga/note-1-v1-8-1-reality-check.md):
 #   * Gemma 4 26B reasoning prefix observed at ~150-300 tokens before
 #     content starts → 1024 covers reasoning + 30-line count comfortably.
 #   * Non-thinking baseline kept conservative-but-non-tight (256/512) to

coderouter/guards/backend_health.py

@@ -0,0 +1,208 @@
+"""Backend health monitor (v1.9-E phase 2, L5).
+
+Passive health state machine for chain providers. Counts consecutive
+failures observed via :meth:`record_attempt(provider, success=False)`
+and transitions the provider's state through:
+
+    HEALTHY  →  DEGRADED  →  UNHEALTHY
+       ▲          │             │
+       └──────────┴─────────────┘
+              success=True
+
+The engine consults :meth:`is_unhealthy(provider)` at chain-resolve
+time and (when the active profile's ``backend_health_action`` is
+``demote``) moves UNHEALTHY providers to the back of the chain, so
+the chain prefers a known-up backend without ever skipping a
+provider entirely. A single subsequent success snaps the provider
+back to HEALTHY immediately — no rolling-window inertia, no debounce.
+
+Why state-machine, not rolling window
+=====================================
+
+The v1.9-C :class:`coderouter.routing.adaptive.AdaptiveAdjuster`
+already handles the **gradient** case (continuous latency / error-
+rate observations, rolling window, debounced demotions). L5 handles
+the **binary** case: did this backend just crash and start refusing
+every request? A hard crash produces a deterministic stream of
+identical errors on every retry — a state machine catches it in
+``threshold`` attempts without waiting for a rolling window to
+saturate.
+
+The two adjusters are orthogonal and compose: the engine consults
+the AdaptiveAdjuster's ``compute_effective_order`` first, then the
+L5 monitor's UNHEALTHY demotion runs on the result. Either signal
+alone is enough to demote a bad provider; both together produce the
+expected chain reorder.
+
+Concurrency
+===========
+
+Thread-safe via an internal ``RLock``. Reads (``state_for`` /
+``is_unhealthy``) and writes (``record_attempt``) hold the lock for
+the body of the operation so a chain resolve and a per-attempt
+record can't observe a torn state.
+"""
+
+from __future__ import annotations
+
+import threading
+from dataclasses import dataclass
+from typing import Literal
+
+HealthState = Literal["HEALTHY", "DEGRADED", "UNHEALTHY"]
+"""Three-class health classification.
+
+- ``HEALTHY``   — initial state; consecutive failure count is 0.
+- ``DEGRADED``  — failure count has reached ``threshold``; the
+                  provider is still attempted in chain order, but
+                  the state-changed log fires for operator visibility.
+- ``UNHEALTHY`` — failure count has reached ``2 * threshold``; when
+                  the profile's action is ``demote``, the chain
+                  resolver moves this provider to the back. A single
+                  success resets back to ``HEALTHY`` directly.
+"""
+
+
+@dataclass(frozen=True)
+class HealthTransition:
+    """The outcome of a state-changing :meth:`record_attempt` call.
+
+    Returned by :meth:`record_attempt` so the engine can decide
+    whether to emit a ``backend-health-changed`` log line. The
+    engine treats ``None`` returns as "no state change" and stays
+    quiet; non-None returns surface a single info line carrying the
+    transition.
+    """
+
+    provider: str
+    old_state: HealthState
+    new_state: HealthState
+    consecutive_failures: int
+
+
+@dataclass
+class _ProviderHealth:
+    state: HealthState = "HEALTHY"
+    consecutive_failures: int = 0
+
+
+class BackendHealthMonitor:
+    """Per-provider health state machine.
+
+    Public API:
+
+    - :meth:`record_attempt(provider, success)` — fold one observed
+      attempt outcome. Returns a :class:`HealthTransition` iff the
+      provider's state changed; ``None`` otherwise.
+    - :meth:`state_for(provider)` — current state (``HEALTHY``
+      default for never-observed providers).
+    - :meth:`is_unhealthy(provider)` — convenience predicate the
+      engine consults at chain-resolve time.
+    - :meth:`reset()` — drop all state. Mainly for tests.
+
+    The threshold parameter is supplied per-call (rather than stored
+    on the monitor) so different profiles in the same engine can use
+    different thresholds without forcing the monitor to be aware of
+    profile resolution. The transition rules are:
+
+      * failure → ``consecutive_failures += 1``
+        - if it reaches ``2 * threshold``: state = UNHEALTHY
+        - elif it reaches ``threshold``: state = DEGRADED
+        - else state unchanged
+      * success → ``consecutive_failures = 0``, state = HEALTHY
+    """
+
+    def __init__(self) -> None:
+        self._lock: threading.RLock = threading.RLock()
+        self._state: dict[str, _ProviderHealth] = {}
+
+    # ------------------------------------------------------------------
+    # Recording
+    # ------------------------------------------------------------------
+
+    def record_attempt(
+        self,
+        provider: str,
+        *,
+        success: bool,
+        threshold: int,
+    ) -> HealthTransition | None:
+        """Fold one observed attempt outcome and return a transition (if any).
+
+        Returns ``None`` when the operation didn't change the
+        provider's state — the engine uses this to gate
+        ``backend-health-changed`` log emissions (no log spam when
+        a HEALTHY provider succeeds repeatedly).
+        """
+        with self._lock:
+            entry = self._state.setdefault(provider, _ProviderHealth())
+            old_state = entry.state
+
+            if success:
+                entry.consecutive_failures = 0
+                if old_state != "HEALTHY":
+                    entry.state = "HEALTHY"
+                    return HealthTransition(
+                        provider=provider,
+                        old_state=old_state,
+                        new_state="HEALTHY",
+                        consecutive_failures=0,
+                    )
+                return None
+
+            entry.consecutive_failures += 1
+            new_state: HealthState
+            if entry.consecutive_failures >= 2 * threshold:
+                new_state = "UNHEALTHY"
+            elif entry.consecutive_failures >= threshold:
+                new_state = "DEGRADED"
+            else:
+                # Below threshold — state unchanged but failure
+                # counter is still ticking (caller may transition
+                # us on a future call).
+                return None
+
+            if new_state == old_state:
+                # We've already been at this level (e.g. already
+                # UNHEALTHY and failing again) — no transition fired.
+                return None
+
+            entry.state = new_state
+            return HealthTransition(
+                provider=provider,
+                old_state=old_state,
+                new_state=new_state,
+                consecutive_failures=entry.consecutive_failures,
+            )
+
+    def reset(self) -> None:
+        """Drop all per-provider state."""
+        with self._lock:
+            self._state.clear()
+
+    # ------------------------------------------------------------------
+    # Read-only API
+    # ------------------------------------------------------------------
+
+    def state_for(self, provider: str) -> HealthState:
+        """Return ``provider``'s current health state.
+
+        Never-observed providers default to ``HEALTHY`` — the chain
+        resolver doesn't have to special-case "first attempt".
+        """
+        with self._lock:
+            entry = self._state.get(provider)
+            if entry is None:
+                return "HEALTHY"
+            return entry.state
+
+    def is_unhealthy(self, provider: str) -> bool:
+        """True iff ``provider``'s current state is ``UNHEALTHY``."""
+        return self.state_for(provider) == "UNHEALTHY"
+
+
+__all__ = [
+    "BackendHealthMonitor",
+    "HealthState",
+    "HealthTransition",
+]

coderouter/guards/memory_pressure.py

@@ -0,0 +1,210 @@
+"""Memory-pressure detection guard (v1.9-E phase 2, L2).
+
+Local backends (Ollama / LM Studio / llama.cpp) report VRAM
+exhaustion via HTTP 5xx with error bodies that include phrases like
+``"out of memory"``, ``"CUDA out of memory"``, ``"insufficient
+memory"``, etc. Without intervention, the engine retries against the
+same backend on the very next request and trips the same OOM — wasted
+latency, wasted tokens (when the failure happens after partial
+generation), and an operator-visible cascade of 5xx in the dashboard.
+
+This module gives the engine two pieces:
+
+  1. A **stateless detector** :func:`is_memory_pressure_error` that
+     decides "is this AdapterError an OOM-coded failure" from the
+     error message text. Pure, no observability dependencies.
+  2. A **stateful tracker** :class:`MemoryPressureGuard` that records
+     "provider X is pressured until ts" entries (TTL-based cooldown)
+     and answers ``is_pressured(provider)`` at chain-resolve time.
+
+The combination lets the engine react to an observed OOM by skipping
+the same provider for ``memory_pressure_cooldown_s`` seconds — the
+chain falls through to the next provider, which is typically a
+lighter-weight model or a remote fallback with the headroom.
+
+Detection patterns
+==================
+
+Case-insensitive substring match against a curated phrase list; no
+regex backtracking. Patterns chosen to match the actual error bodies
+observed across:
+
+  * **Ollama** ``/api/generate`` and ``/v1/chat/completions``:
+    ``"model requires more system memory"``,
+    ``"out of memory"``.
+  * **LM Studio** ``/v1/messages`` and ``/v1/chat/completions``:
+    ``"insufficient memory"``, ``"failed to load model"``.
+  * **llama.cpp** ``llama-server``:
+    ``"failed to allocate"``, ``"CUDA out of memory"``.
+  * Generic CUDA / Metal patterns:
+    ``"out of memory"``, ``"OOM"``.
+
+False-positive risk is low because all patterns require the
+substring "memory" or the literal "OOM" in the failure body — generic
+HTTP errors don't include those words.
+
+Concurrency
+===========
+
+The tracker is safe across asyncio tasks within one event loop and
+across worker threads — every public method holds an ``RLock`` for
+the body of its read/write.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+
+from coderouter.adapters.base import AdapterError
+
+# ---------------------------------------------------------------------------
+# Detection patterns
+# ---------------------------------------------------------------------------
+
+
+# Lowercased substrings — tested against ``str(adapter_error).lower()``.
+# Order doesn't matter (any-match short-circuits). Each entry is a phrase
+# observed in a real backend OOM response. New patterns can be added
+# defensively (false-positive risk is low; see module docstring).
+_MEMORY_PRESSURE_PHRASES: tuple[str, ...] = (
+    "out of memory",
+    "cuda out of memory",
+    "metal out of memory",
+    "insufficient memory",
+    "model requires more system memory",
+    "failed to allocate",
+    "failed to load model",
+    "ggml_cuda_host_malloc",  # llama.cpp specific
+    "oom",
+)
+
+
+def is_memory_pressure_error(exc: AdapterError) -> bool:
+    """Return True iff ``exc`` looks like a backend OOM signal.
+
+    Pure function. Operates on ``str(exc).lower()`` and checks for
+    any of the curated phrases in :data:`_MEMORY_PRESSURE_PHRASES`.
+    Callers in the engine wrap each ``provider-failed`` site to
+    decide whether to mark the provider pressured.
+
+    The check is intentionally **only** about the message text, not
+    the HTTP status. Backends sometimes return 500 vs 503 for OOM
+    inconsistently, and a 500 carrying ``"missing model"`` should
+    NOT be treated as memory pressure (no cooldown helps recover
+    from a config error). The phrase-match keeps the detector
+    focused on the actual signal.
+    """
+    text = str(exc).lower()
+    return any(phrase in text for phrase in _MEMORY_PRESSURE_PHRASES)
+
+
+# ---------------------------------------------------------------------------
+# Tracker
+# ---------------------------------------------------------------------------
+
+
+class MemoryPressureGuard:
+    """In-memory per-provider OOM cooldown tracker.
+
+    Public API:
+
+    - :meth:`mark_pressured(provider, cooldown_s)` — start (or extend)
+      a cooldown window for ``provider``. Idempotent: re-marking a
+      pressured provider extends the deadline to ``now + cooldown_s``.
+    - :meth:`is_pressured(provider)` — True iff the provider's
+      cooldown deadline is in the future. Lazy expiry: when an
+      expired entry is observed, it's swept out so subsequent reads
+      see the entry-less default.
+    - :meth:`pressured_until(provider)` — monotonic timestamp of the
+      cooldown deadline (or 0.0 if not pressured). Useful for log
+      payloads that want to surface the human-readable expiry.
+    - :meth:`reset()` — drop all entries. Mainly for tests.
+
+    Internal lock: an ``RLock`` covers every read/write pair so a
+    concurrent ``mark_pressured`` from a failed call and an
+    ``is_pressured`` from a chain resolve can't observe a torn
+    state.
+
+    Time source: ``time.monotonic`` so cooldowns are immune to
+    wall-clock skew. Tests inject ``now=`` for determinism.
+    """
+
+    def __init__(self) -> None:
+        self._lock: threading.RLock = threading.RLock()
+        self._until: dict[str, float] = {}
+
+    # ------------------------------------------------------------------
+    # Mutating API
+    # ------------------------------------------------------------------
+
+    def mark_pressured(
+        self,
+        provider: str,
+        cooldown_s: float,
+        *,
+        now: float | None = None,
+    ) -> float:
+        """Mark ``provider`` as pressured for ``cooldown_s`` seconds.
+
+        Returns the resulting cooldown deadline (monotonic ts), so
+        callers that want to log the expiry can pull it out without
+        a second locked read. Idempotent — re-marking extends the
+        deadline.
+        """
+        ts = now if now is not None else time.monotonic()
+        deadline = ts + cooldown_s
+        with self._lock:
+            self._until[provider] = deadline
+        return deadline
+
+    def reset(self) -> None:
+        """Drop all cooldown entries immediately."""
+        with self._lock:
+            self._until.clear()
+
+    # ------------------------------------------------------------------
+    # Read-only API
+    # ------------------------------------------------------------------
+
+    def is_pressured(self, provider: str, *, now: float | None = None) -> bool:
+        """True iff ``provider`` is currently in cooldown.
+
+        Lazy expiry: if the recorded deadline has passed, the entry
+        is dropped before this call returns False. Callers don't see
+        stale "pressured" entries.
+        """
+        ts = now if now is not None else time.monotonic()
+        with self._lock:
+            deadline = self._until.get(provider)
+            if deadline is None:
+                return False
+            if deadline <= ts:
+                # Cooldown elapsed — sweep so subsequent calls don't
+                # re-take the lock for an empty entry.
+                del self._until[provider]
+                return False
+            return True
+
+    def pressured_until(
+        self, provider: str, *, now: float | None = None
+    ) -> float:
+        """Return ``provider``'s cooldown deadline, or 0.0 if not pressured.
+
+        Same lazy-expiry behavior as :meth:`is_pressured`.
+        """
+        ts = now if now is not None else time.monotonic()
+        with self._lock:
+            deadline = self._until.get(provider)
+            if deadline is None:
+                return 0.0
+            if deadline <= ts:
+                del self._until[provider]
+                return 0.0
+            return deadline
+
+
+__all__ = [
+    "MemoryPressureGuard",
+    "is_memory_pressure_error",
+]