evalkeep 0.1.0__py3-none-any.whl

evalkeep/analysis.py

@@ -0,0 +1,174 @@
+"""Structured failure analysis and the provider interface behind it.
+
+Detection says *that* a trace failed and points at the evidence. Analysis says
+*what kind* of failure it is, so that similar failures can be grouped and a
+representative can be chosen. That description is a judgement, so every analysis
+records who made it -- a person, or a named model at a named prompt version --
+and keeps the raw response for audit.
+
+The vocabularies below are deliberately closed. Free-text labels do not cluster:
+"wrong order id", "refunded the wrong order" and "bad tool arg" are one failure
+family that three different analysts would name three different ways.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from enum import StrEnum
+from typing import Any, ClassVar, Protocol, runtime_checkable
+
+from evalkeep.detectors import Signal
+from evalkeep.trace import NormalizedTrace
+
+
+class FailureType(StrEnum):
+    """What went wrong."""
+
+    WRONG_TOOL_ARGUMENT = "wrong_tool_argument"
+    WRONG_TOOL_SELECTION = "wrong_tool_selection"
+    MISSING_TOOL_CALL = "missing_tool_call"
+    UNNECESSARY_ACTION = "unnecessary_action"
+    INCORRECT_ANSWER = "incorrect_answer"
+    INCOMPLETE_ANSWER = "incomplete_answer"
+    UNSUPPORTED_CLAIM = "unsupported_claim"
+    FORMAT_VIOLATION = "format_violation"
+    POLICY_VIOLATION = "policy_violation"
+    UNWARRANTED_REFUSAL = "unwarranted_refusal"
+    EXECUTION_ERROR = "execution_error"
+    OTHER = "other"
+
+
+class Component(StrEnum):
+    """Where in the agent it went wrong."""
+
+    PLANNING = "planning"
+    TOOL_SELECTION = "tool_selection"
+    TOOL_ARGUMENTS = "tool_arguments"
+    TOOL_EXECUTION = "tool_execution"
+    RETRIEVAL = "retrieval"
+    RESPONSE_GENERATION = "response_generation"
+    POLICY = "policy"
+    UNKNOWN = "unknown"
+
+
+class Severity(StrEnum):
+    LOW = "low"
+    MEDIUM = "medium"
+    HIGH = "high"
+    CRITICAL = "critical"
+
+
+#: Ordered worst-first, for selecting high-severity representatives in 8F.
+SEVERITY_ORDER: tuple[Severity, ...] = (
+    Severity.CRITICAL,
+    Severity.HIGH,
+    Severity.MEDIUM,
+    Severity.LOW,
+)
+
+
+@dataclass(frozen=True)
+class ProviderAnalysis:
+    """What a provider produced, before the pipeline stamps its provenance."""
+
+    failure_type: FailureType
+    component: Component
+    severity: Severity
+    summary: str
+    raw_response: str | None = None
+
+
+@dataclass
+class FailureAnalysis:
+    """A provider analysis plus the provenance that makes it auditable."""
+
+    failure_type: FailureType
+    component: Component
+    severity: Severity
+    summary: str
+    #: Who decided: ``manual:alex``, ``anthropic:claude-opus-5``, ``stub``.
+    analyzer: str
+    prompt_version: int
+    analyzed_at: datetime = field(default_factory=lambda: datetime.now(UTC))
+    #: Set when a person labelled by hand.
+    labeler: str | None = None
+    #: The provider's own words, redacted. Kept so a surprising label can be
+    #: checked against what the model actually said.
+    raw_response: str | None = None
+
+    @property
+    def manual(self) -> bool:
+        return self.analyzer.startswith("manual")
+
+    @property
+    def severity_rank(self) -> int:
+        return SEVERITY_ORDER.index(self.severity)
+
+    @classmethod
+    def from_provider(
+        cls,
+        produced: ProviderAnalysis,
+        *,
+        analyzer: str,
+        prompt_version: int,
+    ) -> FailureAnalysis:
+        return cls(
+            failure_type=produced.failure_type,
+            component=produced.component,
+            severity=produced.severity,
+            summary=produced.summary,
+            analyzer=analyzer,
+            prompt_version=prompt_version,
+            raw_response=produced.raw_response,
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "failure_type": self.failure_type.value,
+            "component": self.component.value,
+            "severity": self.severity.value,
+            "summary": self.summary,
+            "analyzer": self.analyzer,
+            "prompt_version": self.prompt_version,
+            "analyzed_at": self.analyzed_at.isoformat(),
+            "labeler": self.labeler,
+            "raw_response": self.raw_response,
+        }
+
+    @classmethod
+    def from_dict(cls, payload: dict[str, Any]) -> FailureAnalysis:
+        return cls(
+            failure_type=FailureType(payload["failure_type"]),
+            component=Component(payload["component"]),
+            severity=Severity(payload["severity"]),
+            summary=payload["summary"],
+            analyzer=payload["analyzer"],
+            prompt_version=int(payload["prompt_version"]),
+            analyzed_at=datetime.fromisoformat(payload["analyzed_at"]),
+            labeler=payload.get("labeler"),
+            raw_response=payload.get("raw_response"),
+        )
+
+
+class AnalyzerError(Exception):
+    """A provider could not analyze this failure. Never fatal to a run."""
+
+
+@runtime_checkable
+class AnalyzerProvider(Protocol):
+    """Provider-independent analysis.
+
+    ``identity`` is part of the cache key, so it must change whenever the thing
+    producing the answer changes -- a different model is a different analyst.
+    """
+
+    name: ClassVar[str]
+    description: ClassVar[str]
+
+    @property
+    def identity(self) -> str: ...
+
+    def analyze_failure(self, trace: NormalizedTrace, signals: list[Signal]) -> ProviderAnalysis:
+        """Describe one failure. Raises :class:`AnalyzerError` on failure."""
+        ...

evalkeep/analysis_run.py

@@ -0,0 +1,160 @@
+"""The analysis pass: describe failures, cached, without ever losing a run.
+
+Two properties matter more than speed here:
+
+* **A provider failure is never fatal.** One trace the model chokes on must not
+  abandon the other four hundred. Failures are counted and reported; the run
+  continues.
+* **The provider's own words are kept, redacted.** The model only ever sees a
+  redacted trace, but its response is redacted again before storage -- a model
+  can quote its input, and "it only saw redacted text" is an argument, not a
+  guarantee.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+
+from evalkeep.analysis import AnalyzerError, AnalyzerProvider, FailureAnalysis
+from evalkeep.cache import AnalysisCache, cache_key
+from evalkeep.failures import FailureStatus
+from evalkeep.hashing import content_hash
+from evalkeep.prompts import FAILURE_ANALYSIS_PROMPT_VERSION
+from evalkeep.redaction import RedactionSummary, Redactor
+from evalkeep.storage import TraceStore
+
+
+@dataclass
+class AnalysisReport:
+    """What one analysis pass did."""
+
+    analyzer: str
+    prompt_version: int
+    considered: int = 0
+    analyzed: int = 0
+    from_cache: int = 0
+    skipped: int = 0
+    manual_kept: int = 0
+    failed: int = 0
+    redactions: int = 0
+    errors: list[tuple[str, str]] = field(default_factory=list)
+    by_type: dict[str, int] = field(default_factory=dict)
+
+    @property
+    def changed(self) -> bool:
+        return bool(self.analyzed)
+
+
+#: Analysis describes failures worth keeping; a dismissed one is not.
+ANALYZABLE = (FailureStatus.CANDIDATE, FailureStatus.CONFIRMED)
+
+
+def analyze_failures(
+    store: TraceStore,
+    provider: AnalyzerProvider,
+    cache: AnalysisCache,
+    *,
+    redactor: Redactor | None = None,
+    reanalyze: bool = False,
+    overwrite_manual: bool = False,
+    limit: int | None = None,
+) -> AnalysisReport:
+    """Analyze every failure that needs it, reusing cached answers."""
+    redactor = redactor or Redactor()
+    report = AnalysisReport(
+        analyzer=provider.identity, prompt_version=FAILURE_ANALYSIS_PROMPT_VERSION
+    )
+    failures = store.failures
+
+    for failure in failures.iter_all():
+        if failure.status not in ANALYZABLE:
+            continue
+        if limit is not None and report.analyzed + report.from_cache >= limit:
+            break
+
+        report.considered += 1
+        existing = failures.get_analysis(failure.failure_id)
+        if existing is not None and not _needs_analysis(
+            existing, provider, reanalyze=reanalyze, overwrite_manual=overwrite_manual
+        ):
+            report.skipped += 1
+            if existing.manual:
+                report.manual_kept += 1
+            continue
+
+        stored = store.get(failure.trace_id)
+        if stored is None:  # pragma: no cover - the foreign key prevents this
+            report.failed += 1
+            report.errors.append((failure.failure_id, "trace is missing from the store"))
+            continue
+
+        key = cache_key(
+            content_hash(stored.trace), provider.identity, FAILURE_ANALYSIS_PROMPT_VERSION
+        )
+        cached = cache.get(key)
+        if cached is not None and not reanalyze:
+            analysis = FailureAnalysis.from_dict(cached)
+            analysis.analyzed_at = datetime.now(UTC)
+            failures.save_analysis(failure.failure_id, analysis)
+            report.from_cache += 1
+            _count_type(report, analysis)
+            continue
+
+        try:
+            produced = provider.analyze_failure(stored.trace, failure.signals)
+        except AnalyzerError as exc:
+            report.failed += 1
+            report.errors.append((failure.failure_id, str(exc)))
+            continue
+
+        analysis = FailureAnalysis.from_provider(
+            produced,
+            analyzer=provider.identity,
+            prompt_version=FAILURE_ANALYSIS_PROMPT_VERSION,
+        )
+        report.redactions += _redact_in_place(analysis, redactor)
+
+        failures.save_analysis(failure.failure_id, analysis)
+        cache.put(key, analysis.to_dict())
+        report.analyzed += 1
+        _count_type(report, analysis)
+
+    return report
+
+
+def _needs_analysis(
+    existing: FailureAnalysis,
+    provider: AnalyzerProvider,
+    *,
+    reanalyze: bool,
+    overwrite_manual: bool,
+) -> bool:
+    """Re-analyze when forced, or when the analysis is stale -- never over a label.
+
+    ``--reanalyze`` refreshes *machine* analyses. Replacing something a person
+    wrote takes its own flag: refreshing model output after a prompt change is
+    routine, and it must not quietly discard hand-written labels along the way.
+    """
+    if existing.manual:
+        return overwrite_manual
+    if reanalyze:
+        return True
+    return (
+        existing.analyzer != provider.identity
+        or existing.prompt_version != FAILURE_ANALYSIS_PROMPT_VERSION
+    )
+
+
+def _redact_in_place(analysis: FailureAnalysis, redactor: Redactor) -> int:
+    """Redact what the provider wrote, before it is stored anywhere."""
+    summary = RedactionSummary()
+    analysis.summary = redactor.redact_text(analysis.summary, summary)
+    if analysis.raw_response is not None:
+        analysis.raw_response = redactor.redact_text(analysis.raw_response, summary)
+    return summary.total
+
+
+def _count_type(report: AnalysisReport, analysis: FailureAnalysis) -> None:
+    key = analysis.failure_type.value
+    report.by_type[key] = report.by_type.get(key, 0) + 1

evalkeep/analyzers/__init__.py

@@ -0,0 +1,52 @@
+"""Analyzer providers and the registry the project configuration resolves against.
+
+``manual`` is the default and is deliberately *not* a provider: it means nobody
+analyzes automatically, and failures are labelled by hand. Resolving it returns
+``None`` so callers must handle the offline case explicitly rather than silently
+falling back to a machine label.
+"""
+
+from __future__ import annotations
+
+from evalkeep.analysis import AnalyzerError, AnalyzerProvider
+from evalkeep.analyzers.stub import StubAnalyzer
+from evalkeep.config import AnalyzerConfig
+from evalkeep.errors import CommandError
+
+MANUAL_PROVIDER = "manual"
+
+KNOWN_PROVIDERS: dict[str, str] = {
+    MANUAL_PROVIDER: "No automatic analysis; label failures by hand",
+    "anthropic": "Claude via the Anthropic Messages API (needs an API key)",
+    StubAnalyzer.name: StubAnalyzer.description,
+}
+
+
+def get_analyzer(config: AnalyzerConfig) -> AnalyzerProvider | None:
+    """Build the configured provider, or ``None`` for manual labelling."""
+    if config.provider == MANUAL_PROVIDER:
+        return None
+    if config.provider == StubAnalyzer.name:
+        return StubAnalyzer()
+    if config.provider == "anthropic":
+        # Imported lazily: the SDK is an optional dependency.
+        from evalkeep.analyzers.anthropic import AnthropicAnalyzer
+
+        return AnthropicAnalyzer(
+            model=config.model, effort=config.effort, max_tokens=config.max_tokens
+        )
+    known = ", ".join(sorted(KNOWN_PROVIDERS))
+    raise CommandError(
+        f"Unknown analyzer provider {config.provider!r}.",
+        hint=f"Known providers: {known}.",
+    )
+
+
+__all__ = [
+    "KNOWN_PROVIDERS",
+    "MANUAL_PROVIDER",
+    "AnalyzerError",
+    "AnalyzerProvider",
+    "StubAnalyzer",
+    "get_analyzer",
+]

evalkeep/analyzers/anthropic.py

@@ -0,0 +1,145 @@
+"""The Anthropic analyzer.
+
+Uses the Messages API with structured outputs, so the model is constrained to
+the JSON schema in :mod:`evalkeep.prompts` rather than asked politely for JSON
+and parsed hopefully. The model only ever sees a redacted trace, and the
+pipeline redacts the response again before storing it.
+
+The ``anthropic`` package is an optional dependency: Evalkeep runs entirely
+offline with manual labelling, and nothing in the core imports this module
+unless the project configures ``provider: anthropic``.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any, ClassVar
+
+from pydantic import BaseModel, ValidationError
+
+from evalkeep.analysis import (
+    AnalyzerError,
+    Component,
+    FailureType,
+    ProviderAnalysis,
+    Severity,
+)
+from evalkeep.detectors import Signal
+from evalkeep.prompts import (
+    FAILURE_ANALYSIS_SCHEMA,
+    FAILURE_ANALYSIS_SYSTEM,
+    failure_analysis_prompt,
+)
+from evalkeep.trace import NormalizedTrace
+
+DEFAULT_MODEL = "claude-opus-5"
+DEFAULT_MAX_TOKENS = 16000
+DEFAULT_EFFORT = "medium"
+
+
+class _AnalysisResponse(BaseModel):
+    """The schema the model is constrained to, validated again on arrival."""
+
+    failure_type: FailureType
+    component: Component
+    severity: Severity
+    summary: str
+
+
+class AnthropicAnalyzer:
+    name: ClassVar[str] = "anthropic"
+    description: ClassVar[str] = "Claude via the Anthropic Messages API (needs an API key)"
+
+    def __init__(
+        self,
+        *,
+        model: str = DEFAULT_MODEL,
+        effort: str = DEFAULT_EFFORT,
+        max_tokens: int = DEFAULT_MAX_TOKENS,
+        client: Any = None,
+    ) -> None:
+        self.model = model
+        self.effort = effort
+        self.max_tokens = max_tokens
+        self._client = client
+
+    @property
+    def identity(self) -> str:
+        """A different model is a different analyst, so it keys the cache."""
+        return f"{self.name}:{self.model}"
+
+    @property
+    def client(self) -> Any:
+        if self._client is None:
+            self._client = _build_client()
+        return self._client
+
+    def analyze_failure(self, trace: NormalizedTrace, signals: list[Signal]) -> ProviderAnalysis:
+        try:
+            response = self.client.messages.create(
+                model=self.model,
+                max_tokens=self.max_tokens,
+                system=FAILURE_ANALYSIS_SYSTEM,
+                thinking={"type": "adaptive"},
+                output_config={
+                    "effort": self.effort,
+                    "format": {"type": "json_schema", "schema": FAILURE_ANALYSIS_SCHEMA},
+                },
+                messages=[{"role": "user", "content": failure_analysis_prompt(trace, signals)}],
+            )
+        except Exception as exc:  # SDK exception classes are not importable here
+            raise AnalyzerError(f"{self.identity} request failed: {exc}") from exc
+
+        return _parse(response, self.identity)
+
+
+def _parse(response: Any, identity: str) -> ProviderAnalysis:
+    stop_reason = getattr(response, "stop_reason", None)
+    if stop_reason == "refusal":
+        raise AnalyzerError(f"{identity} declined to analyze this trace")
+    if stop_reason == "max_tokens":
+        raise AnalyzerError(f"{identity} hit max_tokens before finishing")
+
+    text = next(
+        (block.text for block in response.content if getattr(block, "type", None) == "text"),
+        None,
+    )
+    if not text:
+        raise AnalyzerError(f"{identity} returned no text content")
+
+    try:
+        payload = json.loads(text)
+    except json.JSONDecodeError as exc:
+        raise AnalyzerError(f"{identity} returned text that is not JSON: {exc}") from exc
+
+    try:
+        parsed = _AnalysisResponse.model_validate(payload)
+    except ValidationError as exc:
+        raise AnalyzerError(
+            f"{identity} returned JSON that does not match the schema:\n{exc}"
+        ) from exc
+
+    return ProviderAnalysis(
+        failure_type=parsed.failure_type,
+        component=parsed.component,
+        severity=parsed.severity,
+        summary=parsed.summary.strip(),
+        raw_response=text,
+    )
+
+
+def _build_client() -> Any:
+    try:
+        import anthropic
+    except ImportError as exc:
+        raise AnalyzerError(
+            "The 'anthropic' package is not installed. "
+            "Install it with: uv add 'evalkeep[anthropic]'"
+        ) from exc
+    try:
+        return anthropic.Anthropic()
+    except Exception as exc:
+        raise AnalyzerError(
+            f"Could not create an Anthropic client: {exc}. "
+            "Set ANTHROPIC_API_KEY, or run 'ant auth login'."
+        ) from exc

evalkeep/analyzers/stub.py

@@ -0,0 +1,34 @@
+"""A deterministic analyzer for development, with no network and no key.
+
+It exists so the analysis pipeline -- caching, storage, reporting -- can be
+exercised end to end offline. It is not analysis: it reads the detector evidence
+back to you and labels everything ``other``/``unknown``. Its output is stamped
+``stub`` so nothing downstream can mistake it for a real judgement.
+"""
+
+from __future__ import annotations
+
+from typing import ClassVar
+
+from evalkeep.analysis import Component, FailureType, ProviderAnalysis, Severity
+from evalkeep.detectors import Signal
+from evalkeep.trace import NormalizedTrace
+
+
+class StubAnalyzer:
+    name: ClassVar[str] = "stub"
+    description: ClassVar[str] = "Deterministic placeholder for offline development"
+
+    @property
+    def identity(self) -> str:
+        return self.name
+
+    def analyze_failure(self, trace: NormalizedTrace, signals: list[Signal]) -> ProviderAnalysis:
+        summary = "; ".join(signal.summary for signal in signals) or "no evidence recorded"
+        return ProviderAnalysis(
+            failure_type=FailureType.OTHER,
+            component=Component.UNKNOWN,
+            severity=Severity.MEDIUM,
+            summary=f"[stub] {summary}",
+            raw_response=None,
+        )

evalkeep/cache.py

@@ -0,0 +1,122 @@
+"""On-disk cache for analyzer answers.
+
+Keyed by the three things that can change the answer: the *content* of the trace
+(already redacted), which analyst produced it, and which prompt version was
+asked. Change any one and the key changes, so a prompt edit or a model swap
+never serves a stale label -- and re-running analysis after a database reset
+costs nothing.
+
+The cache lives under ``.evalkeep/cache/``, which is not committed. It is a
+speed and money optimisation, never a source of truth: deleting it loses
+nothing that the database does not already hold.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+ANALYSIS_DIRNAME = "analysis"
+EMBEDDING_DIRNAME = "embeddings"
+
+
+def cache_key(content_hash: str, analyzer_identity: str, prompt_version: int) -> str:
+    """A stable key over the three inputs that determine an answer."""
+    material = "\n".join([content_hash, analyzer_identity, str(prompt_version)])
+    return hashlib.sha256(material.encode("utf-8")).hexdigest()
+
+
+@dataclass
+class CacheStats:
+    hits: int = 0
+    misses: int = 0
+    writes: int = 0
+
+
+class JsonFileCache:
+    """A content-addressed JSON file cache. Corrupt entries are treated as misses."""
+
+    dirname: str = "cache"
+
+    def __init__(self, root: Path, *, enabled: bool = True) -> None:
+        self.root = root / self.dirname
+        self.enabled = enabled
+        self.stats = CacheStats()
+
+    def path_for(self, key: str) -> Path:
+        # Sharded by the first two characters so one directory never holds
+        # a hundred thousand files.
+        return self.root / key[:2] / f"{key}.json"
+
+    def get(self, key: str) -> dict[str, Any] | None:
+        if not self.enabled:
+            return None
+        path = self.path_for(key)
+        try:
+            payload = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError):
+            self.stats.misses += 1
+            return None
+        if not isinstance(payload, dict):
+            self.stats.misses += 1
+            return None
+        self.stats.hits += 1
+        return payload
+
+    def put(self, key: str, payload: dict[str, Any]) -> None:
+        if not self.enabled:
+            return
+        path = self.path_for(key)
+        try:
+            path.parent.mkdir(parents=True, exist_ok=True)
+            # Write-then-rename so a crash cannot leave a half-written entry
+            # that would later be read back as a valid answer.
+            temporary = path.with_suffix(".tmp")
+            temporary.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8")
+            temporary.replace(path)
+        except OSError:
+            # A cache is an optimisation; failing to write one is not an error.
+            return
+        self.stats.writes += 1
+
+
+class AnalysisCache(JsonFileCache):
+    """Cached analyzer answers, keyed by trace content, analyst and prompt version."""
+
+    dirname = ANALYSIS_DIRNAME
+
+
+class EmbeddingCache(JsonFileCache):
+    """Cached vectors, keyed by the embedded text and the embedding space.
+
+    Embedding the same failure description twice is wasted work with a local
+    provider and wasted money with a hosted one, and re-running ``discover``
+    after editing one cluster should not re-embed everything.
+    """
+
+    dirname = EMBEDDING_DIRNAME
+
+    def get_vector(self, key: str) -> list[float] | None:
+        payload = self.get(key)
+        if payload is None:
+            return None
+        vector = payload.get("vector")
+        if not isinstance(vector, list) or not all(
+            isinstance(value, int | float) for value in vector
+        ):
+            self.stats.hits -= 1
+            self.stats.misses += 1
+            return None
+        return [float(value) for value in vector]
+
+    def put_vector(self, key: str, vector: list[float]) -> None:
+        self.put(key, {"vector": vector})
+
+
+def embedding_key(text: str, embedder_identity: str) -> str:
+    """A key over the exact text embedded and the space it was embedded into."""
+    material = "\n".join([embedder_identity, text])
+    return hashlib.sha256(material.encode("utf-8")).hexdigest()