coderouter-cli 2.0.0__py3-none-any.whl → 2.2.0__py3-none-any.whl

coderouter/state/audit_log.py

@@ -0,0 +1,269 @@
+"""Structured JSONL audit log (v2.0-K).
+
+Captures guard activations, chain fallbacks, budget warnings,
+self-healing events, and drift transitions as append-only JSONL
+records.  Implements ``logging.Handler`` so it taps the same
+structured log stream that :class:`MetricsCollector` observes — no
+second instrumentation path needed.
+
+Architecture
+============
+
+::
+
+    logger.info("backend-health-changed", extra={...})
+        │
+        ├─ MetricsCollector.emit()   → in-memory counters
+        └─ AuditLogHandler.emit()    → append JSONL line to disk
+
+Only *audit-worthy* events are written (guard state changes, chain
+decisions, cost/budget events, self-healing lifecycle).  High-frequency
+per-request events (``try-provider``, ``provider-ok``) are excluded to
+keep the log small.
+
+File rotation
+=============
+
+Simple single-backup rotation: when the active file exceeds
+``max_bytes``, it is renamed to ``audit.jsonl.1`` (overwriting any
+existing backup) and a fresh ``audit.jsonl`` is started.  One backup
+is enough for the typical use case (reviewing yesterday's events while
+today's stream runs).
+
+Thread safety
+=============
+
+Inherits ``logging.Handler``'s built-in lock (``self.lock``) via the
+``acquire()``/``release()`` protocol.  File writes are atomic single
+lines (no partial-line interleaving) because Python's stdlib file
+``write()`` of a single string ≤ PIPE_BUF is POSIX-atomic on Linux.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from datetime import UTC, datetime
+from pathlib import Path
+
+# Events that are audit-worthy: guard state changes, chain decisions,
+# cost/budget, self-healing lifecycle, drift, probing milestones.
+_AUDIT_EVENTS: frozenset[str] = frozenset(
+    {
+        # Backend health
+        "backend-health-changed",
+        "demote-unhealthy-provider",
+        # Self-healing (v2.0-J)
+        "self-healing-exclude",
+        "self-healing-restore",
+        "self-healing-restart",
+        "self-healing-recovery-probe",
+        # Budget / cost
+        "skip-budget-exceeded",
+        "chain-budget-exceeded",
+        # Chain gate events
+        "chain-paid-gate-blocked",
+        "chain-memory-pressure-blocked",
+        "chain-uniform-auth-failure",
+        # Memory pressure
+        "memory-pressure-detected",
+        # Drift (v2.0-G)
+        "drift-detected",
+        "drift-promoted",
+        "drift-reload-attempted",
+        "drift-recovered",
+        # Context budget (v2.0-F)
+        "context-budget-warning",
+        "context-budget-trimmed",
+        # Tool loop (L3)
+        "tool-loop-detected",
+        # Probe milestones (v2.0-I)
+        "probe-capabilities-drift",
+        # Startup / shutdown
+        "coderouter-startup",
+        "coderouter-shutdown",
+    }
+)
+
+
+class AuditLogHandler(logging.Handler):
+    """Append-only JSONL handler for audit-worthy events.
+
+    Public API:
+
+    - Constructor: ``AuditLogHandler(log_path, max_bytes=10_485_760)``
+    - Inherited ``emit()`` is called automatically by the logging
+      framework for every log record.
+    - :meth:`close()` — flush and close the file handle.
+
+    Only events whose ``record.msg`` is in :data:`_AUDIT_EVENTS` are
+    written.  Everything else is silently ignored (zero I/O cost for
+    non-audit log lines).
+    """
+
+    def __init__(
+        self,
+        log_path: str | Path,
+        *,
+        max_bytes: int = 10_485_760,
+    ) -> None:
+        super().__init__(level=logging.DEBUG)
+        self._log_path = Path(log_path)
+        self._max_bytes = max_bytes
+        self._log_path.parent.mkdir(parents=True, exist_ok=True)
+        self._file = open(self._log_path, "a", encoding="utf-8")  # noqa: SIM115
+
+    def emit(self, record: logging.LogRecord) -> None:
+        """Write an audit line if the event is audit-worthy."""
+        if record.msg not in _AUDIT_EVENTS:
+            return
+        try:
+            self.acquire()
+            try:
+                line = self._format_line(record)
+                self._file.write(line)
+                self._file.flush()
+                self._maybe_rotate()
+            finally:
+                self.release()
+        except Exception:
+            self.handleError(record)
+
+    def close(self) -> None:
+        """Flush and close the underlying file."""
+        self.acquire()
+        try:
+            if self._file and not self._file.closed:
+                self._file.flush()
+                self._file.close()
+        finally:
+            self.release()
+        super().close()
+
+    # ------------------------------------------------------------------
+    # Internals
+    # ------------------------------------------------------------------
+
+    def _format_line(self, record: logging.LogRecord) -> str:
+        """Build a single JSONL line from a log record."""
+        payload: dict[str, object] = {
+            "ts": datetime.now(UTC).isoformat(),
+            "event": record.msg,
+            "level": record.levelname,
+        }
+        # Merge structured extras (skip stdlib internal fields).
+        _stdlib_keys = {
+            "name",
+            "msg",
+            "args",
+            "created",
+            "filename",
+            "funcName",
+            "levelname",
+            "levelno",
+            "lineno",
+            "module",
+            "msecs",
+            "pathname",
+            "process",
+            "processName",
+            "relativeCreated",
+            "stack_info",
+            "thread",
+            "threadName",
+            "exc_info",
+            "exc_text",
+            "message",
+            "taskName",
+        }
+        for key, value in record.__dict__.items():
+            if key.startswith("_") or key in _stdlib_keys:
+                continue
+            payload[key] = value
+        return json.dumps(payload, default=str) + "\n"
+
+    def _maybe_rotate(self) -> None:
+        """Rotate if the current file exceeds max_bytes."""
+        try:
+            size = self._file.tell()
+            if size < self._max_bytes:
+                return
+            self._file.close()
+            backup = self._log_path.with_suffix(".jsonl.1")
+            # Overwrite any existing backup.
+            if backup.exists():
+                backup.unlink()
+            self._log_path.rename(backup)
+            self._file = open(self._log_path, "a", encoding="utf-8")  # noqa: SIM115
+        except OSError:
+            # If rotation fails, just keep writing to the current file.
+            if self._file.closed:
+                self._file = open(self._log_path, "a", encoding="utf-8")  # noqa: SIM115
+
+
+def read_audit_log(
+    log_path: str | Path,
+    *,
+    tail: int | None = None,
+    event_filter: str | None = None,
+    since: str | None = None,
+) -> list[dict[str, object]]:
+    """Read and filter audit log entries.
+
+    Parameters:
+
+    - ``tail`` — return only the last N entries.
+    - ``event_filter`` — only entries whose ``event`` field contains
+      this substring (case-insensitive).
+    - ``since`` — only entries with ``ts >= since`` (ISO 8601 prefix
+      match).
+
+    Returns a list of parsed dicts, newest last.
+    """
+    path = Path(log_path)
+    if not path.exists():
+        return []
+
+    entries: list[dict[str, object]] = []
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                entry = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+
+            if event_filter and event_filter.lower() not in str(
+                entry.get("event", "")
+            ).lower():
+                continue
+
+            if since:
+                ts = str(entry.get("ts", ""))
+                if ts < since:
+                    continue
+
+            entries.append(entry)
+
+    if tail is not None and tail > 0:
+        entries = entries[-tail:]
+
+    return entries
+
+
+def summarize_audit_log(entries: list[dict[str, object]]) -> dict[str, int]:
+    """Return event type → count summary from a list of audit entries."""
+    summary: dict[str, int] = {}
+    for entry in entries:
+        event = str(entry.get("event", "unknown"))
+        summary[event] = summary.get(event, 0) + 1
+    return dict(sorted(summary.items(), key=lambda x: -x[1]))
+
+
+__all__ = [
+    "AuditLogHandler",
+    "read_audit_log",
+    "summarize_audit_log",
+]

coderouter/state/replay.py

@@ -0,0 +1,316 @@
+"""Replay analysis engine (v2.0-K Replay framework).
+
+Provides statistical A/B comparison of request journal entries across
+providers.  Since the request journal records only metadata (token
+counts, cost, streaming flag) — **not** request/response bodies — this
+is *statistical replay*, not literal re-execution.
+
+Typical use: an operator changes the fallback chain (swap provider A
+for provider B) and wants to know how the new routing affected cost,
+token counts, and request distribution compared to the previous window.
+
+Usage::
+
+    from coderouter.state.request_log import read_request_log
+    from coderouter.state.replay import compare_providers, summarize_window
+
+    entries = read_request_log("~/.coderouter/state/requests.jsonl")
+    summary = summarize_window(entries)
+    comparison = compare_providers(entries, "anthropic-api", "openrouter-free")
+"""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass, field
+
+# ------------------------------------------------------------------
+# Per-provider summary
+# ------------------------------------------------------------------
+
+
+@dataclass
+class ProviderSummary:
+    """Aggregated statistics for one provider over a time window."""
+
+    provider: str
+    request_count: int = 0
+    total_input_tokens: int = 0
+    total_output_tokens: int = 0
+    total_cost_usd: float = 0.0
+    total_cost_savings_usd: float = 0.0
+    total_cache_read_tokens: int = 0
+    total_cache_creation_tokens: int = 0
+    streaming_count: int = 0
+    # Derived (populated by _finalize)
+    avg_input_tokens: float = 0.0
+    avg_output_tokens: float = 0.0
+    avg_cost_usd: float = 0.0
+    streaming_ratio: float = 0.0
+    cache_hit_ratio: float = 0.0
+
+    def _finalize(self) -> None:
+        """Compute derived averages and ratios."""
+        n = self.request_count
+        if n > 0:
+            self.avg_input_tokens = self.total_input_tokens / n
+            self.avg_output_tokens = self.total_output_tokens / n
+            self.avg_cost_usd = self.total_cost_usd / n
+            self.streaming_ratio = self.streaming_count / n
+        total_input = self.total_input_tokens + self.total_cache_read_tokens
+        if total_input > 0:
+            self.cache_hit_ratio = self.total_cache_read_tokens / total_input
+
+
+@dataclass
+class WindowSummary:
+    """Full window summary across all providers."""
+
+    total_requests: int = 0
+    total_cost_usd: float = 0.0
+    total_cost_savings_usd: float = 0.0
+    total_input_tokens: int = 0
+    total_output_tokens: int = 0
+    providers: dict[str, ProviderSummary] = field(default_factory=dict)
+    first_ts: str = ""
+    last_ts: str = ""
+
+
+def summarize_window(entries: list[dict[str, object]]) -> WindowSummary:
+    """Aggregate request journal entries into a :class:`WindowSummary`.
+
+    Parameters
+    ----------
+    entries
+        Parsed JSONL dicts from :func:`read_request_log`.
+
+    Returns
+    -------
+    WindowSummary
+        Per-provider and overall statistics.
+    """
+    summary = WindowSummary()
+    for entry in entries:
+        provider = str(entry.get("provider", "unknown"))
+        if provider not in summary.providers:
+            summary.providers[provider] = ProviderSummary(provider=provider)
+
+        ps = summary.providers[provider]
+        ps.request_count += 1
+
+        input_tokens = int(entry.get("input_tokens", 0))
+        output_tokens = int(entry.get("output_tokens", 0))
+        cost_usd = float(entry.get("cost_usd", 0.0))
+        cost_savings = float(entry.get("cost_savings_usd", 0.0))
+        cache_read = int(entry.get("cache_read_input_tokens", 0))
+        cache_creation = int(entry.get("cache_creation_input_tokens", 0))
+        streaming = bool(entry.get("streaming", False))
+
+        ps.total_input_tokens += input_tokens
+        ps.total_output_tokens += output_tokens
+        ps.total_cost_usd += cost_usd
+        ps.total_cost_savings_usd += cost_savings
+        ps.total_cache_read_tokens += cache_read
+        ps.total_cache_creation_tokens += cache_creation
+        if streaming:
+            ps.streaming_count += 1
+
+        summary.total_requests += 1
+        summary.total_cost_usd += cost_usd
+        summary.total_cost_savings_usd += cost_savings
+        summary.total_input_tokens += input_tokens
+        summary.total_output_tokens += output_tokens
+
+        ts = str(entry.get("ts", ""))
+        if ts:
+            if not summary.first_ts or ts < summary.first_ts:
+                summary.first_ts = ts
+            if not summary.last_ts or ts > summary.last_ts:
+                summary.last_ts = ts
+
+    for ps in summary.providers.values():
+        ps._finalize()
+
+    return summary
+
+
+# ------------------------------------------------------------------
+# A/B provider comparison
+# ------------------------------------------------------------------
+
+
+@dataclass
+class ProviderComparison:
+    """Side-by-side comparison of two providers."""
+
+    provider_a: ProviderSummary
+    provider_b: ProviderSummary
+    # Deltas: B - A (positive = B is larger)
+    delta_avg_input_tokens: float = 0.0
+    delta_avg_output_tokens: float = 0.0
+    delta_avg_cost_usd: float = 0.0
+    delta_total_cost_usd: float = 0.0
+    # Percentage changes (relative to A; NaN if A is zero)
+    pct_avg_cost_change: float = 0.0
+    pct_total_cost_change: float = 0.0
+
+
+def compare_providers(
+    entries: list[dict[str, object]],
+    provider_a: str,
+    provider_b: str,
+) -> ProviderComparison:
+    """Compare two providers' statistics from the same journal.
+
+    Parameters
+    ----------
+    entries
+        Parsed JSONL dicts from :func:`read_request_log`.
+    provider_a, provider_b
+        Provider names to compare. Entries not matching either are
+        ignored.
+
+    Returns
+    -------
+    ProviderComparison
+        Side-by-side stats with deltas.
+    """
+    a_entries = [e for e in entries if str(e.get("provider", "")) == provider_a]
+    b_entries = [e for e in entries if str(e.get("provider", "")) == provider_b]
+
+    a_summary = summarize_window(a_entries)
+    b_summary = summarize_window(b_entries)
+
+    ps_a = a_summary.providers.get(
+        provider_a, ProviderSummary(provider=provider_a)
+    )
+    ps_b = b_summary.providers.get(
+        provider_b, ProviderSummary(provider=provider_b)
+    )
+
+    comparison = ProviderComparison(provider_a=ps_a, provider_b=ps_b)
+    comparison.delta_avg_input_tokens = ps_b.avg_input_tokens - ps_a.avg_input_tokens
+    comparison.delta_avg_output_tokens = ps_b.avg_output_tokens - ps_a.avg_output_tokens
+    comparison.delta_avg_cost_usd = ps_b.avg_cost_usd - ps_a.avg_cost_usd
+    comparison.delta_total_cost_usd = ps_b.total_cost_usd - ps_a.total_cost_usd
+
+    if ps_a.avg_cost_usd > 0:
+        comparison.pct_avg_cost_change = (
+            (ps_b.avg_cost_usd - ps_a.avg_cost_usd) / ps_a.avg_cost_usd * 100.0
+        )
+    else:
+        comparison.pct_avg_cost_change = float("nan")
+
+    if ps_a.total_cost_usd > 0:
+        comparison.pct_total_cost_change = (
+            (ps_b.total_cost_usd - ps_a.total_cost_usd) / ps_a.total_cost_usd * 100.0
+        )
+    else:
+        comparison.pct_total_cost_change = float("nan")
+
+    return comparison
+
+
+# ------------------------------------------------------------------
+# CLI table formatting helpers
+# ------------------------------------------------------------------
+
+
+def format_summary_table(summary: WindowSummary) -> str:
+    """Render a :class:`WindowSummary` as a CLI table.
+
+    Returns a plain-text table suitable for terminal output.
+    """
+    lines: list[str] = []
+    lines.append(f"Window: {summary.first_ts} → {summary.last_ts}")
+    lines.append(f"Total: {summary.total_requests} requests, "
+                 f"${summary.total_cost_usd:.4f} cost, "
+                 f"${summary.total_cost_savings_usd:.4f} savings")
+    lines.append("")
+
+    # Header
+    hdr = (
+        f"{'Provider':<25} {'Reqs':>6} {'AvgIn':>8} {'AvgOut':>8} "
+        f"{'AvgCost':>10} {'TotalCost':>10} {'Cache%':>7} {'Stream%':>8}"
+    )
+    lines.append(hdr)
+    lines.append("-" * len(hdr))
+
+    for ps in sorted(summary.providers.values(),
+                     key=lambda p: p.total_cost_usd, reverse=True):
+        lines.append(
+            f"{ps.provider:<25} {ps.request_count:>6} "
+            f"{ps.avg_input_tokens:>8.0f} {ps.avg_output_tokens:>8.0f} "
+            f"${ps.avg_cost_usd:>9.4f} ${ps.total_cost_usd:>9.4f} "
+            f"{ps.cache_hit_ratio * 100:>6.1f}% "
+            f"{ps.streaming_ratio * 100:>7.1f}%"
+        )
+
+    return "\n".join(lines)
+
+
+def format_comparison_table(comp: ProviderComparison) -> str:
+    """Render a :class:`ProviderComparison` as a CLI table.
+
+    Returns a plain-text side-by-side comparison table.
+    """
+    a = comp.provider_a
+    b = comp.provider_b
+    lines: list[str] = []
+
+    hdr = f"{'Metric':<25} {a.provider:<20} {b.provider:<20} {'Delta':>12}"
+    lines.append(hdr)
+    lines.append("-" * len(hdr))
+
+    def _row(label: str, va: object, vb: object, delta: float, fmt: str = ".0f") -> str:
+        d_str = f"{delta:+{fmt}}"
+        return f"{label:<25} {va!s:<20} {vb!s:<20} {d_str:>12}"
+
+    lines.append(_row("Requests", a.request_count, b.request_count,
+                       b.request_count - a.request_count))
+    lines.append(_row("Avg input tokens", f"{a.avg_input_tokens:.0f}",
+                       f"{b.avg_input_tokens:.0f}",
+                       comp.delta_avg_input_tokens))
+    lines.append(_row("Avg output tokens", f"{a.avg_output_tokens:.0f}",
+                       f"{b.avg_output_tokens:.0f}",
+                       comp.delta_avg_output_tokens))
+    lines.append(_row("Avg cost (USD)", f"${a.avg_cost_usd:.4f}",
+                       f"${b.avg_cost_usd:.4f}",
+                       comp.delta_avg_cost_usd, fmt=".4f"))
+    lines.append(_row("Total cost (USD)", f"${a.total_cost_usd:.4f}",
+                       f"${b.total_cost_usd:.4f}",
+                       comp.delta_total_cost_usd, fmt=".4f"))
+    lines.append(_row("Cache hit ratio", f"{a.cache_hit_ratio * 100:.1f}%",
+                       f"{b.cache_hit_ratio * 100:.1f}%",
+                       (b.cache_hit_ratio - a.cache_hit_ratio) * 100, fmt=".1f"))
+    lines.append(_row("Streaming ratio", f"{a.streaming_ratio * 100:.1f}%",
+                       f"{b.streaming_ratio * 100:.1f}%",
+                       (b.streaming_ratio - a.streaming_ratio) * 100, fmt=".1f"))
+
+    # Cost change summary
+    lines.append("")
+    if not math.isnan(comp.pct_avg_cost_change):
+        direction = "cheaper" if comp.pct_avg_cost_change < 0 else "more expensive"
+        lines.append(
+            f"Per-request: {b.provider} is {abs(comp.pct_avg_cost_change):.1f}% "
+            f"{direction} than {a.provider}"
+        )
+    if not math.isnan(comp.pct_total_cost_change):
+        direction = "less" if comp.pct_total_cost_change < 0 else "more"
+        lines.append(
+            f"Total spend: {b.provider} spent {abs(comp.pct_total_cost_change):.1f}% "
+            f"{direction} than {a.provider}"
+        )
+
+    return "\n".join(lines)
+
+
+__all__ = [
+    "ProviderComparison",
+    "ProviderSummary",
+    "WindowSummary",
+    "compare_providers",
+    "format_comparison_table",
+    "format_summary_table",
+    "summarize_window",
+]