coderouter-cli 2.0.0__py3-none-any.whl → 2.2.0__py3-none-any.whl

coderouter/ingress/app.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import contextlib
 import os
 from collections.abc import AsyncIterator
 from contextlib import asynccontextmanager
@@ -71,7 +72,135 @@ def create_app(config_path: str | None = None) -> FastAPI:
         # chronological order. Non-fatal — the chain still works, just
         # potentially sub-optimally for the agentic harness.
         check_claude_code_chain_suitability(config, logger=logger)
+
+        # v2.0-K: attach persistent state store + audit/request log if configured.
+        state_store = None
+        audit_handler = None
+        request_log_handler = None
+        if config.state_dir:
+            import logging as _logging
+            from pathlib import Path
+
+            from coderouter.state.audit_log import AuditLogHandler
+            from coderouter.state.store import StateStore
+
+            state_path = Path(config.state_dir).expanduser()
+            state_store = StateStore(state_path / "coderouter.db")
+            engine.attach_state_store(state_store)
+
+            # Restore MetricsCollector state from the store.
+            from coderouter.metrics import get_collector
+
+            collector = get_collector()
+            if collector is not None:
+                metrics_state = state_store.get("metrics", "state")
+                if metrics_state is not None:
+                    with contextlib.suppress(Exception):
+                        collector.load_state(metrics_state)  # type: ignore[arg-type]
+
+            logger.info(
+                "state-store-attached",
+                extra={"state_dir": str(state_path)},
+            )
+
+            if config.audit_log == "active":
+                audit_handler = AuditLogHandler(
+                    state_path / "audit.jsonl",
+                    max_bytes=config.audit_log_max_bytes,
+                )
+                _logging.getLogger().addHandler(audit_handler)
+                logger.info(
+                    "audit-log-started",
+                    extra={
+                        "path": str(state_path / "audit.jsonl"),
+                        "max_bytes": config.audit_log_max_bytes,
+                    },
+                )
+
+            if config.request_log == "active":
+                from coderouter.state.request_log import RequestLogHandler
+
+                request_log_handler = RequestLogHandler(
+                    state_path / "requests.jsonl",
+                    max_bytes=config.request_log_max_bytes,
+                )
+                _logging.getLogger().addHandler(request_log_handler)
+                logger.info(
+                    "request-log-started",
+                    extra={
+                        "path": str(state_path / "requests.jsonl"),
+                        "max_bytes": config.request_log_max_bytes,
+                    },
+                )
+
+        # v2.0-I: launch continuous probe background task if configured.
+        probe_task = None
+        shutdown_event = None
+        if config.continuous_probe == "active":
+            import asyncio
+
+            from coderouter.guards.continuous_probe import probe_loop
+            from coderouter.routing.capability import get_default_registry
+
+            shutdown_event = asyncio.Event()
+            probe_task = asyncio.create_task(
+                probe_loop(
+                    config.providers,
+                    record_fn=engine.backend_health.record_attempt,
+                    interval_s=config.probe_interval_s,
+                    timeout_s=config.probe_timeout_s,
+                    probe_paid=config.probe_paid,
+                    shutdown_event=shutdown_event,
+                    registry=get_default_registry(),
+                )
+            )
+            logger.info(
+                "continuous-probe-started",
+                extra={
+                    "interval_s": config.probe_interval_s,
+                    "probe_paid": config.probe_paid,
+                    "providers": len(config.providers),
+                },
+            )
+
         yield
+
+        # Graceful shutdown of probe task
+        if probe_task is not None and shutdown_event is not None:
+            shutdown_event.set()
+            with contextlib.suppress(Exception):
+                await probe_task
+
+        # v2.0-J: graceful shutdown of recovery probe tasks.
+        with contextlib.suppress(Exception):
+            await engine.shutdown_recovery_probes()
+
+        # v2.0-K: persist state and close audit log on shutdown.
+        if state_store is not None:
+            with contextlib.suppress(Exception):
+                engine.save_all_state()
+            # Save MetricsCollector state.
+            from coderouter.metrics import get_collector
+
+            collector = get_collector()
+            if collector is not None:
+                with contextlib.suppress(Exception):
+                    state_store.put("metrics", "state", collector.save_state())
+            with contextlib.suppress(Exception):
+                state_store.close()
+        if audit_handler is not None:
+            import logging as _logging
+
+            with contextlib.suppress(Exception):
+                _logging.getLogger().removeHandler(audit_handler)
+                audit_handler.close()
+        if request_log_handler is not None:
+            import logging as _logging
+
+            with contextlib.suppress(Exception):
+                _logging.getLogger().removeHandler(request_log_handler)
+                request_log_handler.close()
+
         logger.info("coderouter-shutdown")
 
     app = FastAPI(

coderouter/logging.py

@@ -595,6 +595,114 @@ def log_demote_unhealthy_provider(
     logger.info("demote-unhealthy-provider", extra=payload)
 
 
+# ---------------------------------------------------------------------------
+# v2.0-J: self-healing log shapes
+# ---------------------------------------------------------------------------
+
+
+class SelfHealingExcludePayload(TypedDict):
+    """Structured shape of the ``self-healing-exclude`` log record."""
+
+    provider: str
+    profile: str
+    consecutive_failures: int
+
+
+class SelfHealingRestorePayload(TypedDict):
+    """Structured shape of the ``self-healing-restore`` log record."""
+
+    provider: str
+    profile: str
+    excluded_duration_s: float
+
+
+class SelfHealingRestartPayload(TypedDict):
+    """Structured shape of the ``self-healing-restart`` log record."""
+
+    provider: str
+    command: str
+    success: bool
+    error: str | None
+
+
+class SelfHealingRecoveryProbePayload(TypedDict):
+    """Structured shape of the ``self-healing-recovery-probe`` log record."""
+
+    provider: str
+    success: bool
+    next_interval_s: float
+    latency_ms: float
+
+
+def log_self_healing_exclude(
+    logger: logging.Logger,
+    *,
+    provider: str,
+    profile: str,
+    consecutive_failures: int,
+) -> None:
+    """Emit when a provider is excluded from the chain by self-healing."""
+    payload: SelfHealingExcludePayload = {
+        "provider": provider,
+        "profile": profile,
+        "consecutive_failures": consecutive_failures,
+    }
+    logger.warning("self-healing-exclude", extra=payload)
+
+
+def log_self_healing_restore(
+    logger: logging.Logger,
+    *,
+    provider: str,
+    profile: str,
+    excluded_duration_s: float,
+) -> None:
+    """Emit when a previously excluded provider is restored to the chain."""
+    payload: SelfHealingRestorePayload = {
+        "provider": provider,
+        "profile": profile,
+        "excluded_duration_s": round(excluded_duration_s, 1),
+    }
+    logger.info("self-healing-restore", extra=payload)
+
+
+def log_self_healing_restart(
+    logger: logging.Logger,
+    *,
+    provider: str,
+    command: str,
+    success: bool,
+    error: str | None = None,
+) -> None:
+    """Emit after attempting to restart a provider's backend process."""
+    payload: SelfHealingRestartPayload = {
+        "provider": provider,
+        "command": command,
+        "success": success,
+        "error": error,
+    }
+    level = logging.INFO if success else logging.WARNING
+    logger.log(level, "self-healing-restart", extra=payload)
+
+
+def log_self_healing_recovery_probe(
+    logger: logging.Logger,
+    *,
+    provider: str,
+    success: bool,
+    next_interval_s: float,
+    latency_ms: float,
+) -> None:
+    """Emit after each recovery probe attempt for an excluded provider."""
+    payload: SelfHealingRecoveryProbePayload = {
+        "provider": provider,
+        "success": success,
+        "next_interval_s": round(next_interval_s, 1),
+        "latency_ms": round(latency_ms, 1),
+    }
+    logger.info("self-healing-recovery-probe", extra=payload)
+
+
 # ---------------------------------------------------------------------------
 # v1.0-A: output-filter-applied log shape
 #
@@ -1101,3 +1209,265 @@ def log_context_budget_trimmed(
         "max_context_tokens": max_context_tokens,
     }
     logger.info("context-budget-trimmed", extra=payload)
+
+
+# ---------------------------------------------------------------------------
+# v2.0-G (L4): Drift detection logging
+# ---------------------------------------------------------------------------
+
+
+def log_drift_detected(
+    logger: logging.Logger,
+    *,
+    provider: str,
+    profile: str,
+    severity: str,
+    reason: str,
+    action: str,
+    signals: dict[str, float],
+) -> None:
+    """Emit a ``drift-detected`` warning line.
+
+    Fired when the drift detector finds quality degradation in the
+    provider's rolling response window.
+    """
+    logger.warning(
+        "drift-detected",
+        extra={
+            "provider": provider,
+            "profile": profile,
+            "severity": severity,
+            "reason": reason,
+            "action": action,
+            "signals": signals,
+        },
+    )
+
+
+def log_drift_promoted(
+    logger: logging.Logger,
+    *,
+    provider: str,
+    profile: str,
+    demoted_to_rank: int,
+    cooldown_s: int,
+) -> None:
+    """Emit a ``drift-promoted`` info line.
+
+    Fired when a drifted provider is demoted in the chain and a
+    different provider takes over as primary.
+    """
+    logger.info(
+        "drift-promoted",
+        extra={
+            "provider": provider,
+            "profile": profile,
+            "demoted_to_rank": demoted_to_rank,
+            "cooldown_s": cooldown_s,
+        },
+    )
+
+
+def log_drift_reload_attempted(
+    logger: logging.Logger,
+    *,
+    provider: str,
+    success: bool,
+) -> None:
+    """Emit a ``drift-reload-attempted`` info line.
+
+    Fired after attempting an Ollama KV cache flush (keep_alive=0).
+    """
+    logger.info(
+        "drift-reload-attempted",
+        extra={
+            "provider": provider,
+            "success": success,
+        },
+    )
+
+
+def log_drift_recovered(
+    logger: logging.Logger,
+    *,
+    provider: str,
+    profile: str,
+    after_s: float,
+) -> None:
+    """Emit a ``drift-recovered`` info line.
+
+    Fired when a previously-drifted provider's cooldown expires and its
+    rank is restored.
+    """
+    logger.info(
+        "drift-recovered",
+        extra={
+            "provider": provider,
+            "profile": profile,
+            "after_s": round(after_s, 1),
+        },
+    )
+
+
+# ---------------------------------------------------------------------------
+# v2.0-H (L6): Partial stitch surfaced logging
+# ---------------------------------------------------------------------------
+
+
+def log_partial_stitch_surfaced(
+    logger: logging.Logger,
+    *,
+    provider: str,
+    profile: str,
+    text_blocks: int,
+    text_length: int,
+) -> None:
+    """Emit a ``partial-stitch-surfaced`` info line.
+
+    Fired when a mid-stream failure is gracefully terminated with partial
+    content delivered to the client (partial_stitch_action=surface).
+    """
+    logger.info(
+        "partial-stitch-surfaced",
+        extra={
+            "provider": provider,
+            "profile": profile,
+            "text_blocks": text_blocks,
+            "text_length": text_length,
+        },
+    )
+
+
+# ---------------------------------------------------------------------------
+# v2.0-I: Continuous probe log shapes
+#
+# Two event lanes mirror the backend-health triplet:
+#   * ``probe-completed``       — info: a single provider probe finished
+#                                  (success or failure). Quiet in normal
+#                                  operation; operators grep for these to
+#                                  diagnose individual backend issues.
+#   * ``probe-round-completed`` — info: one full sweep across all probed
+#                                  providers finished. Summary counter for
+#                                  dashboards to render "probes/min" and
+#                                  "failure ratio per round".
+# ---------------------------------------------------------------------------
+
+
+class ProbeCompletedPayload(TypedDict):
+    """Structured shape of the ``probe-completed`` log record.
+
+    Fields
+        provider: the provider that was probed.
+        success: whether the 1-token probe request succeeded.
+        latency_ms: round-trip time of the probe in milliseconds.
+        error: short error message (truncated) when success=False;
+            None on success.
+        model_name: model name extracted from the probe response,
+            if available.
+    """
+
+    provider: str
+    success: bool
+    latency_ms: float
+    error: str | None
+    model_name: str | None
+
+
+class ProbeRoundCompletedPayload(TypedDict):
+    """Structured shape of the ``probe-round-completed`` log record.
+
+    Fields
+        providers_probed: number of providers probed in this round.
+        failures: number of probe failures in this round.
+    """
+
+    providers_probed: int
+    failures: int
+
+
+def log_probe_completed(
+    logger: logging.Logger,
+    *,
+    provider: str,
+    success: bool,
+    latency_ms: float,
+    error: str | None = None,
+    model_name: str | None = None,
+) -> None:
+    """Emit a ``probe-completed`` info line for one provider probe.
+
+    Single chokepoint mirroring :func:`log_capability_degraded`. Info
+    level — individual probes are diagnostic noise at normal operation;
+    operators filter on ``success=False`` for alerting.
+    """
+    payload: ProbeCompletedPayload = {
+        "provider": provider,
+        "success": success,
+        "latency_ms": round(latency_ms, 1),
+        "error": error,
+        "model_name": model_name,
+    }
+    logger.info("probe-completed", extra=payload)
+
+
+def log_probe_round_completed(
+    logger: logging.Logger,
+    *,
+    providers_probed: int,
+    failures: int,
+) -> None:
+    """Emit a ``probe-round-completed`` info line summarizing one sweep.
+
+    Info level — fires once per probe interval (default 60s). Dashboards
+    render this as "probes/min" and "failure ratio" without needing to
+    aggregate the individual ``probe-completed`` lines.
+    """
+    payload: ProbeRoundCompletedPayload = {
+        "providers_probed": providers_probed,
+        "failures": failures,
+    }
+    logger.info("probe-round-completed", extra=payload)
+
+
+class ProbeCapabilitiesDriftPayload(TypedDict):
+    """Structured shape of the ``probe-capabilities-drift`` log record.
+
+    Fields
+        provider: the provider whose probe response model mismatched.
+        configured_model: the model string in providers.yaml.
+        observed_model: the model name returned by the probe response.
+        in_registry: whether the observed model has an entry in the
+            capability registry.
+    """
+
+    provider: str
+    configured_model: str
+    observed_model: str
+    in_registry: bool
+
+
+def log_probe_capabilities_drift(
+    logger: logging.Logger,
+    *,
+    provider: str,
+    configured_model: str,
+    observed_model: str,
+    in_registry: bool,
+) -> None:
+    """Emit a ``probe-capabilities-drift`` warning line.
+
+    Fired when the continuous probe detects that the model name returned
+    by the provider differs from the configured model. This can indicate
+    a model swap (Ollama auto-updated), a misconfiguration, or a new
+    model that the capability registry doesn't know about yet.
+
+    Warn level — model drift can cause subtle behavior changes (e.g. a
+    model that supports thinking being replaced by one that doesn't).
+    """
+    payload: ProbeCapabilitiesDriftPayload = {
+        "provider": provider,
+        "configured_model": configured_model,
+        "observed_model": observed_model,
+        "in_registry": in_registry,
+    }
+    logger.warning("probe-capabilities-drift", extra=payload)