evalkeep 0.1.0__py3-none-any.whl

evalkeep/detection.py

@@ -0,0 +1,94 @@
+"""The detection pass: turn stored traces into evidence-backed failure candidates.
+
+Detection is idempotent, and the rules that make it so are the whole design:
+
+* A failure's ID is a pure function of its trace ID, so a second pass addresses
+  the same row instead of creating another one.
+* Signals are *derived*. Every pass recomputes and replaces them, so changing or
+  adding a detector updates the evidence on existing failures.
+* A review is *not* derived. Once a person confirms or dismisses a failure,
+  detection updates its signals and leaves the decision, reviewer and reason
+  alone -- automated analysis never overwrites a human judgement.
+* A candidate nobody has reviewed, whose evidence has since disappeared (a
+  detector was removed or narrowed), is withdrawn. Nothing human is lost,
+  because nothing human was there. Manually added failures are never withdrawn.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass, field
+
+from evalkeep.detectors import DETECTORS, FailureDetector, SignalKind, detect_signals
+from evalkeep.failures import Failure, FailureOrigin
+from evalkeep.storage import TraceStore
+
+
+@dataclass
+class DetectionReport:
+    """What one detection pass did."""
+
+    traces: int = 0
+    created: int = 0
+    updated: int = 0
+    unchanged: int = 0
+    withdrawn: int = 0
+    preserved_reviews: int = 0
+    signals: int = 0
+    by_kind: dict[SignalKind, int] = field(default_factory=dict)
+
+    @property
+    def failures(self) -> int:
+        """Traces that came out of this pass carrying evidence."""
+        return self.created + self.updated + self.unchanged
+
+    @property
+    def changed(self) -> bool:
+        return bool(self.created or self.updated or self.withdrawn)
+
+
+def detect_failures(
+    store: TraceStore, *, detectors: Iterable[FailureDetector] = DETECTORS
+) -> DetectionReport:
+    """Run every detector over every stored trace and persist the result."""
+    detectors = tuple(detectors)
+    report = DetectionReport()
+    failures = store.failures
+
+    for trace in store.iter_traces():
+        report.traces += 1
+        signals = detect_signals(trace, detectors)
+        existing = failures.get_by_trace(trace.trace_id)
+
+        if not signals:
+            # No evidence. Withdraw an unreviewed detector candidate; leave
+            # anything a person touched or created.
+            if (
+                existing is not None
+                and not existing.reviewed
+                and existing.origin is FailureOrigin.DETECTOR
+            ):
+                failures.delete(existing.failure_id)
+                report.withdrawn += 1
+            continue
+
+        report.signals += len(signals)
+        for signal in signals:
+            report.by_kind[signal.kind] = report.by_kind.get(signal.kind, 0) + 1
+
+        if existing is None:
+            failures.save(Failure.from_signals(trace.trace_id, signals))
+            report.created += 1
+            continue
+
+        if existing.reviewed:
+            report.preserved_reviews += 1
+        if [s.to_dict() for s in existing.signals] == [s.to_dict() for s in signals]:
+            report.unchanged += 1
+            continue
+
+        existing.signals = signals
+        failures.save(existing)
+        report.updated += 1
+
+    return report

evalkeep/detectors.py

@@ -0,0 +1,182 @@
+"""Failure detectors: evidence that something went wrong, never a guess.
+
+A detector reads one stored (already redacted) trace and emits a
+:class:`Signal` for each piece of explicit evidence it finds. Detectors do not
+infer failure from the shape of an interaction, and they do not score their
+confidence -- a signal either points at something a person recorded, or it does
+not exist. Combining signals is therefore counting evidence, not adding
+probabilities: a trace with three signals is better documented than one with a
+single signal, not "more likely" to be a failure.
+
+Adding a detector means implementing :class:`FailureDetector` and registering
+it; nothing else in the pipeline changes.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Iterator
+from dataclasses import dataclass, field
+from enum import StrEnum
+from typing import Any, ClassVar, Protocol, runtime_checkable
+
+from evalkeep.trace import EvaluationEvent, NormalizedTrace, OutcomeStatus
+
+
+class SignalKind(StrEnum):
+    """The category of evidence, independent of which detector found it."""
+
+    EXPLICIT_STATUS = "explicit_status"
+    NEGATIVE_FEEDBACK = "negative_feedback"
+    FAILED_EVALUATOR = "failed_evaluator"
+
+
+@dataclass(frozen=True)
+class Signal:
+    """One piece of evidence, with a pointer back to where it came from."""
+
+    detector: str
+    kind: SignalKind
+    #: Path into the trace, so a reviewer can check the claim themselves.
+    source: str
+    summary: str
+    evidence: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "detector": self.detector,
+            "kind": self.kind.value,
+            "source": self.source,
+            "summary": self.summary,
+            "evidence": self.evidence,
+        }
+
+    @classmethod
+    def from_dict(cls, payload: dict[str, Any]) -> Signal:
+        return cls(
+            detector=payload["detector"],
+            kind=SignalKind(payload["kind"]),
+            source=payload["source"],
+            summary=payload["summary"],
+            evidence=payload.get("evidence", {}),
+        )
+
+
+@runtime_checkable
+class FailureDetector(Protocol):
+    """Reads a trace, yields evidence. Never raises on well-formed input."""
+
+    name: ClassVar[str]
+    description: ClassVar[str]
+
+    def detect(self, trace: NormalizedTrace) -> Iterable[Signal]:
+        """Yield one signal per piece of evidence found in ``trace``."""
+        ...
+
+
+class ExplicitStatusDetector:
+    """The recorded outcome says the interaction failed."""
+
+    name: ClassVar[str] = "explicit_status"
+    description: ClassVar[str] = "outcome.status is 'failure' or 'error'"
+
+    _FAILING: ClassVar[dict[OutcomeStatus, str]] = {
+        OutcomeStatus.FAILURE: "the trace is explicitly marked as failed",
+        OutcomeStatus.ERROR: "the trace is explicitly marked as errored",
+    }
+
+    def detect(self, trace: NormalizedTrace) -> Iterator[Signal]:
+        summary = self._FAILING.get(trace.outcome.status)
+        if summary is None:
+            return
+        yield Signal(
+            detector=self.name,
+            kind=SignalKind.EXPLICIT_STATUS,
+            source="outcome.status",
+            summary=summary,
+            evidence={"status": trace.outcome.status.value},
+        )
+
+
+class NegativeFeedbackDetector:
+    """Structured feedback records a negative rating.
+
+    Only an explicit ``rating`` of ``negative`` counts. A bare numeric score
+    arrives without its scale -- 2 could be poor out of 5 or good out of 3 --
+    and inventing a threshold would be exactly the kind of blind scoring this
+    pipeline is meant to avoid.
+    """
+
+    name: ClassVar[str] = "negative_feedback"
+    description: ClassVar[str] = "outcome.feedback.rating is 'negative'"
+
+    def detect(self, trace: NormalizedTrace) -> Iterator[Signal]:
+        feedback = trace.outcome.feedback
+        if feedback is None or feedback.rating != "negative":
+            return
+        evidence: dict[str, Any] = {"rating": feedback.rating}
+        if feedback.comment:
+            evidence["comment"] = feedback.comment
+        if feedback.score is not None:
+            evidence["score"] = feedback.score
+        yield Signal(
+            detector=self.name,
+            kind=SignalKind.NEGATIVE_FEEDBACK,
+            source="outcome.feedback",
+            summary=feedback.comment or "feedback was rated negative",
+            evidence=evidence,
+        )
+
+
+class FailedEvaluatorDetector:
+    """An evaluator recorded a failing verdict, in the outcome or in an event."""
+
+    name: ClassVar[str] = "failed_evaluator"
+    description: ClassVar[str] = "an evaluation recorded passed=false"
+
+    def detect(self, trace: NormalizedTrace) -> Iterator[Signal]:
+        for index, evaluation in enumerate(trace.outcome.evaluations):
+            if evaluation.passed is False:
+                yield self._signal(
+                    source=f"outcome.evaluations.{index}",
+                    name=evaluation.name,
+                    reason=evaluation.reason,
+                    score=evaluation.score,
+                )
+        for index, event in enumerate(trace.events):
+            if isinstance(event, EvaluationEvent) and event.passed is False:
+                yield self._signal(
+                    source=f"events.{index}",
+                    name=event.name,
+                    reason=event.reason,
+                    score=event.score,
+                )
+
+    def _signal(self, *, source: str, name: str, reason: str | None, score: float | None) -> Signal:
+        evidence: dict[str, Any] = {"evaluator": name, "passed": False}
+        if reason:
+            evidence["reason"] = reason
+        if score is not None:
+            evidence["score"] = score
+        return Signal(
+            detector=self.name,
+            kind=SignalKind.FAILED_EVALUATOR,
+            source=source,
+            summary=reason or f"evaluator {name!r} reported a failure",
+            evidence=evidence,
+        )
+
+
+#: The detectors run by ``evalkeep detect``, in a fixed order so that the
+#: signals persisted for a trace are reproducible.
+DETECTORS: tuple[FailureDetector, ...] = (
+    ExplicitStatusDetector(),
+    NegativeFeedbackDetector(),
+    FailedEvaluatorDetector(),
+)
+
+
+def detect_signals(
+    trace: NormalizedTrace, detectors: Iterable[FailureDetector] = DETECTORS
+) -> list[Signal]:
+    """Every signal every detector finds in ``trace``, in detector order."""
+    return [signal for detector in detectors for signal in detector.detect(trace)]

evalkeep/discovery.py

@@ -0,0 +1,208 @@
+"""The discovery pass: embed analyzed failures, group them, pick representatives.
+
+Discovery is derived from analysis, so it can always be recomputed. What cannot
+be recomputed is what a reviewer did to the result -- renaming a family,
+dismissing one, merging two that the distance metric kept apart. Those edits are
+preserved where the group survives unchanged, and re-clustering refuses to
+discard them without an explicit ``--force``.
+
+Cluster identity is derived from membership, which is what makes that possible:
+re-running on unchanged data produces the same cluster IDs, so labels stay
+attached to the families they were written for.
+"""
+
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass, field
+
+from evalkeep.cache import EmbeddingCache, embedding_key
+from evalkeep.clustering import (
+    ClusterInput,
+    build_clusters,
+    clustering_parameters,
+    observation_text,
+)
+from evalkeep.clusters import Cluster, ClusteringRun
+from evalkeep.config import ClusteringConfig
+from evalkeep.embeddings import EmbeddingProvider
+from evalkeep.failures import FailureStatus
+from evalkeep.storage import TraceStore
+
+#: Dismissed failures are not part of any family worth covering.
+CLUSTERABLE = (FailureStatus.CANDIDATE, FailureStatus.CONFIRMED)
+
+
+@dataclass
+class DiscoveryReport:
+    """What one discovery pass did."""
+
+    embedder: str
+    considered: int = 0
+    unanalyzed: int = 0
+    clusters: int = 0
+    singletons: int = 0
+    representatives: int = 0
+    embedded: int = 0
+    from_cache: int = 0
+    kept_labels: int = 0
+    discarded_edits: int = 0
+    #: Failures grouped by observed behaviour because nobody described them.
+    grouped_undescribed: int = 0
+    largest: int = 0
+    parameters: dict[str, object] = field(default_factory=dict)
+
+
+class ClusterEditsWouldBeLost(Exception):
+    """Re-clustering would discard reviewer edits that cannot be carried over."""
+
+    def __init__(self, clusters: list[Cluster]) -> None:
+        self.clusters = clusters
+        super().__init__(f"{len(clusters)} edited clusters would be discarded")
+
+
+def discover(
+    store: TraceStore,
+    embedder: EmbeddingProvider,
+    cache: EmbeddingCache,
+    config: ClusteringConfig,
+    *,
+    force: bool = False,
+    group_undescribed: bool = False,
+) -> DiscoveryReport:
+    """Cluster analyzed failures and select representatives."""
+    report = DiscoveryReport(embedder=embedder.identity)
+    report.parameters = dict(clustering_parameters(config))
+
+    inputs = _gather(store, report, group_undescribed=group_undescribed)
+    vectors = _embed(inputs, embedder, cache, report)
+    clusters = build_clusters(inputs, vectors, config)
+
+    _carry_over_edits(store, clusters, report, force=force)
+
+    run = ClusteringRun(
+        run_id=uuid.uuid4().hex,
+        embedder=embedder.identity,
+        dimensions=embedder.dimensions,
+        parameters=report.parameters,
+        failures=len(inputs),
+    )
+    store.clusters.replace_run(run, clusters)
+
+    report.clusters = len(clusters)
+    report.singletons = sum(1 for cluster in clusters if cluster.size == 1)
+    report.representatives = sum(len(cluster.representatives) for cluster in clusters)
+    report.largest = max((cluster.size for cluster in clusters), default=0)
+    return report
+
+
+def _gather(
+    store: TraceStore, report: DiscoveryReport, *, group_undescribed: bool = False
+) -> list[ClusterInput]:
+    """Undismissed failures, grouped by description where one exists.
+
+    An undescribed failure is skipped by default: grouping by observed behaviour
+    is weaker than grouping by what a failure *is*, and silently mixing the two
+    would make a report mean two different things. ``group_undescribed`` opts
+    in, and the counts say which was used.
+    """
+    inputs: list[ClusterInput] = []
+    for failure in store.failures.iter_all():
+        if failure.status not in CLUSTERABLE:
+            continue
+        report.considered += 1
+        analysis = store.failures.get_analysis(failure.failure_id)
+        if analysis is not None:
+            inputs.append(ClusterInput.from_analysis(failure.failure_id, analysis))
+            continue
+
+        report.unanalyzed += 1
+        if not group_undescribed:
+            continue
+        stored = store.get(failure.trace_id)
+        if stored is None:  # pragma: no cover - the foreign key prevents this
+            continue
+        tools = sorted({call.tool for call in stored.trace.tool_calls})
+        inputs.append(
+            ClusterInput.from_observation(
+                failure.failure_id,
+                behaviour=", ".join(tools) or None,
+                text=observation_text(
+                    [call.tool for call in stored.trace.tool_calls],
+                    [signal.kind.value for signal in failure.signals],
+                    [signal.summary for signal in failure.signals],
+                ),
+            )
+        )
+        report.grouped_undescribed += 1
+    # A stable input order keeps the clustering reproducible.
+    inputs.sort(key=lambda item: item.failure_id)
+    return inputs
+
+
+def _embed(
+    inputs: list[ClusterInput],
+    embedder: EmbeddingProvider,
+    cache: EmbeddingCache,
+    report: DiscoveryReport,
+) -> list[list[float]]:
+    """Embed, reusing cached vectors for text that has not changed."""
+    vectors: list[list[float] | None] = []
+    pending: list[int] = []
+
+    for index, item in enumerate(inputs):
+        cached = cache.get_vector(embedding_key(item.text, embedder.identity))
+        vectors.append(cached)
+        if cached is None:
+            pending.append(index)
+
+    if pending:
+        fresh = embedder.embed([inputs[index].text for index in pending])
+        for index, vector in zip(pending, fresh, strict=True):
+            vectors[index] = vector
+            cache.put_vector(embedding_key(inputs[index].text, embedder.identity), vector)
+
+    report.embedded = len(pending)
+    report.from_cache = len(inputs) - len(pending)
+    return [vector for vector in vectors if vector is not None]
+
+
+def _carry_over_edits(
+    store: TraceStore,
+    clusters: list[Cluster],
+    report: DiscoveryReport,
+    *,
+    force: bool,
+) -> None:
+    """Reattach reviewer edits to families that came back unchanged.
+
+    A cluster ID is a function of its members, so a family whose membership did
+    not change keeps its ID -- and with it, its name and its dismissal. A family
+    whose membership *did* change is genuinely a different group, and its edits
+    cannot be carried over honestly. Rather than dropping them quietly, the pass
+    refuses until the caller says to proceed.
+    """
+    existing = {cluster.cluster_id: cluster for cluster in store.clusters.list()}
+    if not existing:
+        return
+
+    rebuilt = {cluster.cluster_id: cluster for cluster in clusters}
+    for cluster_id, previous in existing.items():
+        if not previous.edited:
+            continue
+        survivor = rebuilt.get(cluster_id)
+        if survivor is None:
+            report.discarded_edits += 1
+            continue
+        survivor.label = previous.label
+        survivor.labelled_by = previous.labelled_by
+        survivor.dismissed = previous.dismissed
+        report.kept_labels += 1
+
+    if report.discarded_edits and not force:
+        lost = [
+            cluster
+            for cluster_id, cluster in existing.items()
+            if cluster.edited and cluster_id not in rebuilt
+        ]
+        raise ClusterEditsWouldBeLost(lost)

evalkeep/embeddings/__init__.py

@@ -0,0 +1,31 @@
+"""Embedding providers and the registry the project configuration resolves against."""
+
+from __future__ import annotations
+
+from evalkeep.config import ClusteringConfig
+from evalkeep.embeddings.base import EmbeddingProvider
+from evalkeep.embeddings.hashing import HashingEmbedder
+from evalkeep.errors import CommandError
+
+KNOWN_EMBEDDERS: dict[str, str] = {HashingEmbedder.name: HashingEmbedder.description}
+
+
+def get_embedder(config: ClusteringConfig) -> EmbeddingProvider:
+    """Build the configured embedding provider.
+
+    A hosted embedding model plugs in here: implement
+    :class:`~evalkeep.embeddings.base.EmbeddingProvider`, give it an identity
+    that names the model, and register it below. Nothing downstream changes --
+    the cache and the stored clustering parameters key off ``identity``, so old
+    and new vectors can never be silently mixed.
+    """
+    if config.embedder == HashingEmbedder.name:
+        return HashingEmbedder(dimensions=config.dimensions, seed=config.seed)
+    known = ", ".join(sorted(KNOWN_EMBEDDERS))
+    raise CommandError(
+        f"Unknown embedding provider {config.embedder!r}.",
+        hint=f"Known providers: {known}.",
+    )
+
+
+__all__ = ["KNOWN_EMBEDDERS", "EmbeddingProvider", "HashingEmbedder", "get_embedder"]

evalkeep/embeddings/base.py

@@ -0,0 +1,32 @@
+"""The embedding provider contract.
+
+Embeddings turn failure descriptions into vectors so that similar failures land
+near each other. The interface is deliberately tiny -- one method over a batch
+of strings -- so a hosted embedding model can replace the built-in one without
+anything else in the pipeline changing.
+
+``identity`` is part of every cache key and is stored with every clustering run.
+It must change whenever the vectors would change: a different model, a different
+dimensionality or a different seed is a different embedding space, and mixing
+two of them silently would produce meaningless distances.
+"""
+
+from __future__ import annotations
+
+from typing import ClassVar, Protocol, runtime_checkable
+
+
+@runtime_checkable
+class EmbeddingProvider(Protocol):
+    name: ClassVar[str]
+    description: ClassVar[str]
+
+    @property
+    def identity(self) -> str: ...
+
+    @property
+    def dimensions(self) -> int: ...
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        """Embed a batch of texts, returning one L2-normalized vector each."""
+        ...

evalkeep/embeddings/hashing.py

@@ -0,0 +1,98 @@
+"""A deterministic, offline embedding provider using the hashing trick.
+
+Tokens (and adjacent token pairs) are hashed to fixed positions in a vector with
+a signed contribution, then the vector is L2-normalized so cosine similarity is
+a dot product. This is feature hashing, a real technique -- not a stub -- but it
+is worth being precise about what it does and does not capture:
+
+* It captures **lexical** overlap. Two summaries describing the same mistake in
+  similar words land close together. Bigrams mean word order carries some
+  weight, so "refunded the oldest order" and "ordered the oldest refund" differ.
+* It does **not** capture meaning. "refunded the wrong order" and "issued a
+  credit for the incorrect purchase" share almost no tokens and will not group,
+  where a trained embedding model would place them together.
+
+That tradeoff is acceptable here because the text being embedded is not free
+prose: it is a structured analysis whose failure type and component are part of
+the string, written under a prompt that explicitly asks for wording that repeats
+across the same failure family. Lexical similarity does real work on that input.
+
+Determinism is the point. ``hash()`` is randomized per process in Python and
+must never be used; BLAKE2b keyed by the configured seed gives the same vector
+on every machine and every run, which is what makes a clustering reproducible.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import math
+import re
+from itertools import pairwise
+from typing import ClassVar
+
+DEFAULT_DIMENSIONS = 512
+DEFAULT_SEED = 0
+
+_TOKEN_PATTERN = re.compile(r"[a-z0-9]+")
+
+
+class HashingEmbedder:
+    name: ClassVar[str] = "hashing"
+    description: ClassVar[str] = "Deterministic offline feature hashing (lexical similarity)"
+
+    def __init__(self, *, dimensions: int = DEFAULT_DIMENSIONS, seed: int = DEFAULT_SEED) -> None:
+        if dimensions < 8:
+            raise ValueError("dimensions must be at least 8")
+        self._dimensions = dimensions
+        self.seed = seed
+
+    @property
+    def identity(self) -> str:
+        return f"{self.name}:{self._dimensions}:{self.seed}"
+
+    @property
+    def dimensions(self) -> int:
+        return self._dimensions
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        return [self._embed_one(text) for text in texts]
+
+    def _embed_one(self, text: str) -> list[float]:
+        vector = [0.0] * self._dimensions
+        for feature, weight in _features(text).items():
+            index, sign = self._position(feature)
+            vector[index] += sign * weight
+        return _normalize(vector)
+
+    def _position(self, feature: str) -> tuple[int, int]:
+        """Hash a feature to a bucket and a sign.
+
+        The sign halves the bias from collisions: two unrelated features landing
+        in the same bucket are as likely to cancel as to reinforce.
+        """
+        digest = hashlib.blake2b(
+            feature.encode("utf-8"), digest_size=9, key=str(self.seed).encode("utf-8")
+        ).digest()
+        value = int.from_bytes(digest, "big")
+        return value % self._dimensions, 1 if (value >> 63) & 1 else -1
+
+
+def _features(text: str) -> dict[str, float]:
+    """Unigrams and bigrams, with sublinear term weighting."""
+    tokens = _TOKEN_PATTERN.findall(text.lower())
+    counts: dict[str, int] = {}
+    for token in tokens:
+        counts[token] = counts.get(token, 0) + 1
+    for first, second in pairwise(tokens):
+        bigram = f"{first}_{second}"
+        counts[bigram] = counts.get(bigram, 0) + 1
+    # 1 + log(count): a word repeated ten times matters more than once, but not
+    # ten times more, so one long quoted string cannot dominate a summary.
+    return {feature: 1.0 + math.log(count) for feature, count in counts.items()}
+
+
+def _normalize(vector: list[float]) -> list[float]:
+    magnitude = math.sqrt(sum(value * value for value in vector))
+    if magnitude == 0.0:
+        return vector
+    return [value / magnitude for value in vector]

evalkeep/errors.py

@@ -0,0 +1,42 @@
+"""Stable exit codes and the error type that carries them.
+
+The codes are part of the CLI contract and are relied on by scripts and CI:
+
+* ``0`` -- the command succeeded.
+* ``1`` -- the command ran, but some records were invalid or rejected.
+* ``2`` -- the command itself could not run (bad usage, missing file,
+  unwritable directory, uninitialized project).
+"""
+
+from __future__ import annotations
+
+from enum import IntEnum
+
+
+class ExitCode(IntEnum):
+    OK = 0
+    RECORD_ERRORS = 1
+    COMMAND_ERROR = 2
+
+
+class EvalkeepError(Exception):
+    """An error that maps to a stable exit code and a readable message."""
+
+    exit_code: ExitCode = ExitCode.COMMAND_ERROR
+
+    def __init__(self, message: str, *, hint: str | None = None) -> None:
+        super().__init__(message)
+        self.message = message
+        self.hint = hint
+
+
+class CommandError(EvalkeepError):
+    """The command could not run at all."""
+
+    exit_code = ExitCode.COMMAND_ERROR
+
+
+class RecordError(EvalkeepError):
+    """The command ran but some input records were rejected."""
+
+    exit_code = ExitCode.RECORD_ERRORS