evalkeep 0.1.0__py3-none-any.whl

evalkeep/runs.py

@@ -0,0 +1,185 @@
+"""Evaluation runs and their per-test results.
+
+The distinction that matters here is between a test that *failed* and a test
+that never got to run. An assertion failure is information about the agent; a
+timeout or a crashed provider is information about the harness. Conflating them
+would let an outage look like a regression, which is exactly the mistake the
+comparison stage exists to avoid.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from enum import StrEnum
+from typing import Any
+
+
+class Outcome(StrEnum):
+    PASS = "pass"
+    FAIL = "fail"
+    #: Never evaluated: the harness or the target failed, not the agent's answer.
+    ERROR = "error"
+
+
+class ErrorKind(StrEnum):
+    TIMEOUT = "timeout"
+    EXECUTION_ERROR = "execution_error"
+
+
+class RunStatus(StrEnum):
+    COMPLETED = "completed"
+    FAILED = "failed"
+
+
+class Verdict(StrEnum):
+    """What a whole set of repetitions says about one case.
+
+    The third value is the point of repeating at all: a case that sometimes
+    passes has told you something a single execution cannot.
+    """
+
+    PASS = "pass"
+    FAIL = "fail"
+    #: Passed some repetitions and failed others.
+    FLAKY = "flaky"
+    #: Never actually evaluated.
+    ERROR = "error"
+
+
+@dataclass
+class CaseResult:
+    test_id: str
+    outcome: Outcome
+    error_kind: ErrorKind | None = None
+    error: str | None = None
+    latency_ms: int | None = None
+    #: What the agent actually produced, redacted before storage.
+    observation: str | None = None
+    failed_assertions: list[str] = field(default_factory=list)
+    #: Which execution of this case this was, zero-based.
+    repetition: int = 0
+
+    @property
+    def comparable(self) -> bool:
+        """Errors are reported separately, never counted as pass or fail."""
+        return self.outcome is not Outcome.ERROR
+
+
+@dataclass
+class CaseSummary:
+    """Every execution of one case in one run, taken together."""
+
+    test_id: str
+    passed: int = 0
+    failed: int = 0
+    errored: int = 0
+
+    @property
+    def repetitions(self) -> int:
+        return self.passed + self.failed + self.errored
+
+    @property
+    def evaluated(self) -> int:
+        """Repetitions that actually ran. Errors are not evidence either way."""
+        return self.passed + self.failed
+
+    @property
+    def pass_rate(self) -> float | None:
+        return self.passed / self.evaluated if self.evaluated else None
+
+    @property
+    def verdict(self) -> Verdict:
+        if not self.evaluated:
+            return Verdict.ERROR
+        if self.passed == self.evaluated:
+            return Verdict.PASS
+        if self.passed == 0:
+            return Verdict.FAIL
+        return Verdict.FLAKY
+
+    @property
+    def flaky(self) -> bool:
+        return self.verdict is Verdict.FLAKY
+
+    @property
+    def confidence(self) -> tuple[float, float] | None:
+        """A Wilson interval on the pass rate, or ``None`` with nothing to go on.
+
+        Wilson rather than the normal approximation because these counts are
+        small and one-sided -- 20 passes out of 20 would otherwise produce an
+        interval of zero width, which claims certainty no sample can give.
+        """
+        return wilson_interval(self.passed, self.evaluated)
+
+
+@dataclass
+class EvaluationRun:
+    run_id: str
+    target_id: str
+    #: Identifies the exact set of approved tests that ran, so two runs are only
+    #: compared when they answered the same questions.
+    suite_hash: str
+    tests: int = 0
+    #: How many times each test was executed.
+    repetitions: int = 1
+    status: RunStatus = RunStatus.COMPLETED
+    runner: str | None = None
+    environment: dict[str, Any] = field(default_factory=dict)
+    started_at: datetime = field(default_factory=lambda: datetime.now(UTC))
+    finished_at: datetime | None = None
+    output_dir: str | None = None
+
+
+@dataclass
+class BaselinePromotion:
+    """A recorded decision that one run is now the reference point."""
+
+    promotion_id: str
+    run_id: str
+    target_id: str
+    reviewer: str
+    reason: str | None = None
+    promoted_at: datetime = field(default_factory=lambda: datetime.now(UTC))
+
+
+def suite_hash(test_ids: list[str]) -> str:
+    """A fingerprint of which tests a run covered."""
+    material = "\n".join(sorted(test_ids))
+    return "sha256:" + hashlib.sha256(material.encode("utf-8")).hexdigest()[:16]
+
+
+#: 95% two-sided normal quantile, for the Wilson interval.
+_Z = 1.959963984540054
+
+
+def wilson_interval(successes: int, trials: int) -> tuple[float, float] | None:
+    """A Wilson score interval for a proportion.
+
+    Chosen over the Wald interval because it behaves at the edges, where these
+    measurements live: an all-pass or all-fail case still gets an interval that
+    reflects how few times it was tried.
+    """
+    if trials <= 0:
+        return None
+    rate = successes / trials
+    denominator = 1 + _Z**2 / trials
+    centre = (rate + _Z**2 / (2 * trials)) / denominator
+    spread = _Z * ((rate * (1 - rate) / trials + _Z**2 / (4 * trials**2)) ** 0.5) / denominator
+    return max(0.0, centre - spread), min(1.0, centre + spread)
+
+
+def summarize(results: list[CaseResult]) -> dict[str, CaseSummary]:
+    """Group repeated executions by case."""
+    summaries: dict[str, CaseSummary] = {}
+    for result in results:
+        summary = summaries.setdefault(result.test_id, CaseSummary(test_id=result.test_id))
+        match result.outcome:
+            case Outcome.PASS:
+                summary.passed += 1
+            case Outcome.FAIL:
+                summary.failed += 1
+            case Outcome.ERROR:
+                summary.errored += 1
+    return summaries

evalkeep/storage/__init__.py

@@ -0,0 +1,37 @@
+"""Redacted trace storage: SQLite schema, migrations and the trace store."""
+
+from __future__ import annotations
+
+from evalkeep.storage.clusters import ClusterStore
+from evalkeep.storage.failures import FailureStore, FailureSummary
+from evalkeep.storage.migrations import LATEST_VERSION, MIGRATIONS, Migration, apply_migrations
+from evalkeep.storage.regression import RegressionStore
+from evalkeep.storage.runs import RunStore
+from evalkeep.storage.store import (
+    OccurrenceStats,
+    StoredTrace,
+    StoreOutcome,
+    StoreResult,
+    TraceOccurrence,
+    TraceStore,
+    TraceSummary,
+)
+
+__all__ = [
+    "LATEST_VERSION",
+    "MIGRATIONS",
+    "ClusterStore",
+    "FailureStore",
+    "FailureSummary",
+    "Migration",
+    "OccurrenceStats",
+    "RegressionStore",
+    "RunStore",
+    "StoreOutcome",
+    "StoreResult",
+    "StoredTrace",
+    "TraceOccurrence",
+    "TraceStore",
+    "TraceSummary",
+    "apply_migrations",
+]

evalkeep/storage/clusters.py

@@ -0,0 +1,163 @@
+"""Persistence for clusterings, clusters and their members."""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+from datetime import datetime
+from typing import Any
+
+from evalkeep.clusters import Cluster, ClusteringRun, ClusterMember, MemberRole
+
+
+class ClusterStore:
+    """Read/write access to the clustering tables.
+
+    Only one clustering is current at a time: ``discover`` writes a new run and
+    the previous one is replaced. Keeping several would mean every downstream
+    command had to ask which grouping it meant.
+    """
+
+    def __init__(self, connection: sqlite3.Connection) -> None:
+        self._connection = connection
+
+    def replace_run(self, run: ClusteringRun, clusters: list[Cluster]) -> None:
+        """Install a clustering, atomically, discarding any previous one."""
+        with self._connection:
+            self._connection.execute("DELETE FROM clustering_runs")
+            self._connection.execute(
+                """
+                INSERT INTO clustering_runs (
+                    run_id, created_at, embedder, dimensions, parameters, failures
+                ) VALUES (?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    run.run_id,
+                    run.created_at.isoformat(),
+                    run.embedder,
+                    run.dimensions,
+                    json.dumps(run.parameters, sort_keys=True),
+                    run.failures,
+                ),
+            )
+            for cluster in clusters:
+                self._insert_cluster(cluster, run.run_id)
+
+    def current_run(self) -> ClusteringRun | None:
+        row = self._connection.execute(
+            "SELECT * FROM clustering_runs ORDER BY created_at DESC LIMIT 1"
+        ).fetchone()
+        if row is None:
+            return None
+        return ClusteringRun(
+            run_id=row["run_id"],
+            embedder=row["embedder"],
+            dimensions=row["dimensions"],
+            parameters=json.loads(row["parameters"]),
+            failures=row["failures"],
+            created_at=datetime.fromisoformat(row["created_at"]),
+        )
+
+    def save(self, cluster: Cluster) -> None:
+        """Update one cluster in place, replacing its membership."""
+        run = self.current_run()
+        if run is None:  # pragma: no cover - callers check first
+            raise ValueError("no clustering to update")
+        with self._connection:
+            self._connection.execute(
+                "DELETE FROM clusters WHERE cluster_id = ?", (cluster.cluster_id,)
+            )
+            self._insert_cluster(cluster, run.run_id)
+
+    def delete(self, cluster_id: str) -> None:
+        with self._connection:
+            self._connection.execute("DELETE FROM clusters WHERE cluster_id = ?", (cluster_id,))
+
+    def get(self, cluster_id: str) -> Cluster | None:
+        row = self._connection.execute(
+            "SELECT * FROM clusters WHERE cluster_id = ?", (cluster_id.strip(),)
+        ).fetchone()
+        return self._build(row) if row is not None else None
+
+    def list(self, *, include_dismissed: bool = True) -> list[Cluster]:
+        query = "SELECT * FROM clusters"
+        if not include_dismissed:
+            query += " WHERE dismissed = 0"
+        rows = self._connection.execute(query).fetchall()
+        clusters = [self._build(row) for row in rows]
+        clusters.sort(key=lambda cluster: (-cluster.size, cluster.cluster_id))
+        return clusters
+
+    def count(self) -> int:
+        return int(self._connection.execute("SELECT COUNT(*) AS n FROM clusters").fetchone()["n"])
+
+    def find_by_failure(self, failure_id: str) -> Cluster | None:
+        row = self._connection.execute(
+            """
+            SELECT c.* FROM clusters c
+            JOIN cluster_members m ON m.cluster_id = c.cluster_id
+            WHERE m.failure_id = ?
+            """,
+            (failure_id,),
+        ).fetchone()
+        return self._build(row) if row is not None else None
+
+    def _insert_cluster(self, cluster: Cluster, run_id: str) -> None:
+        self._connection.execute(
+            """
+            INSERT INTO clusters (
+                cluster_id, run_id, label, labelled_by, dismissed, created_at
+            ) VALUES (?, ?, ?, ?, ?, ?)
+            """,
+            (
+                cluster.cluster_id,
+                run_id,
+                cluster.label,
+                cluster.labelled_by,
+                int(cluster.dismissed),
+                cluster.created_at.isoformat(),
+            ),
+        )
+        self._connection.executemany(
+            """
+            INSERT INTO cluster_members (cluster_id, failure_id, distance, roles)
+            VALUES (?, ?, ?, ?)
+            """,
+            [
+                (
+                    cluster.cluster_id,
+                    member.failure_id,
+                    member.distance,
+                    json.dumps([role.value for role in member.roles]),
+                )
+                for member in cluster.members
+            ],
+        )
+
+    def _build(self, row: sqlite3.Row) -> Cluster:
+        return Cluster(
+            cluster_id=row["cluster_id"],
+            label=row["label"],
+            members=_load_members(self._connection, row["cluster_id"]),
+            labelled_by=row["labelled_by"],
+            dismissed=bool(row["dismissed"]),
+            created_at=datetime.fromisoformat(row["created_at"]),
+        )
+
+
+def _load_members(connection: sqlite3.Connection, cluster_id: str) -> list[ClusterMember]:
+    rows = connection.execute(
+        "SELECT * FROM cluster_members WHERE cluster_id = ? ORDER BY distance, failure_id",
+        (cluster_id,),
+    ).fetchall()
+    members: list[ClusterMember] = []
+    for row in rows:
+        raw: Any = json.loads(row["roles"])
+        members.append(
+            ClusterMember(
+                failure_id=row["failure_id"],
+                distance=row["distance"],
+                roles=[MemberRole(value) for value in raw],
+            )
+        )
+    return members

evalkeep/storage/failures.py

@@ -0,0 +1,254 @@
+"""Persistence for failure candidates and their signals.
+
+Shares the trace store's connection, so a failure and its signals are always
+written in the same transaction as each other and under the same foreign keys
+as the trace they describe.
+"""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+from collections.abc import Iterator
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any
+
+from evalkeep.analysis import FailureAnalysis
+from evalkeep.detectors import Signal
+from evalkeep.failures import Failure, FailureOrigin, FailureStatus
+
+
+@dataclass(frozen=True)
+class FailureSummary:
+    """One row of ``evalkeep failures list``."""
+
+    failure_id: str
+    trace_id: str
+    status: FailureStatus
+    origin: FailureOrigin
+    signals: int
+    kinds: list[str]
+    reviewer: str | None
+    failure_type: str | None = None
+    severity: str | None = None
+
+
+class FailureStore:
+    """Read/write access to the failure tables."""
+
+    def __init__(self, connection: sqlite3.Connection) -> None:
+        self._connection = connection
+
+    def save(self, failure: Failure) -> None:
+        """Write a failure and replace its signals, in one transaction."""
+        with self._connection:
+            self._connection.execute(
+                """
+                INSERT INTO failures (
+                    failure_id, trace_id, status, origin, detected_at,
+                    updated_at, reviewer, reason
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(failure_id) DO UPDATE SET
+                    status = excluded.status,
+                    origin = excluded.origin,
+                    updated_at = excluded.updated_at,
+                    reviewer = excluded.reviewer,
+                    reason = excluded.reason
+                """,
+                (
+                    failure.failure_id,
+                    failure.trace_id,
+                    failure.status.value,
+                    failure.origin.value,
+                    failure.detected_at.isoformat(),
+                    failure.updated_at.isoformat(),
+                    failure.reviewer,
+                    failure.reason,
+                ),
+            )
+            self._connection.execute(
+                "DELETE FROM failure_signals WHERE failure_id = ?", (failure.failure_id,)
+            )
+            self._connection.executemany(
+                """
+                INSERT INTO failure_signals (
+                    failure_id, position, detector, kind, source, summary, evidence
+                ) VALUES (?, ?, ?, ?, ?, ?, ?)
+                """,
+                [
+                    (
+                        failure.failure_id,
+                        position,
+                        signal.detector,
+                        signal.kind.value,
+                        signal.source,
+                        signal.summary,
+                        json.dumps(signal.evidence, sort_keys=True),
+                    )
+                    for position, signal in enumerate(failure.signals)
+                ],
+            )
+
+    def save_analysis(self, failure_id: str, analysis: FailureAnalysis) -> None:
+        """Store the latest analysis for a failure, replacing any previous one."""
+        with self._connection:
+            self._connection.execute(
+                """
+                INSERT INTO failure_analyses (
+                    failure_id, failure_type, component, severity, summary,
+                    analyzer, prompt_version, analyzed_at, labeler, raw_response
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(failure_id) DO UPDATE SET
+                    failure_type = excluded.failure_type,
+                    component = excluded.component,
+                    severity = excluded.severity,
+                    summary = excluded.summary,
+                    analyzer = excluded.analyzer,
+                    prompt_version = excluded.prompt_version,
+                    analyzed_at = excluded.analyzed_at,
+                    labeler = excluded.labeler,
+                    raw_response = excluded.raw_response
+                """,
+                (
+                    failure_id,
+                    analysis.failure_type.value,
+                    analysis.component.value,
+                    analysis.severity.value,
+                    analysis.summary,
+                    analysis.analyzer,
+                    analysis.prompt_version,
+                    analysis.analyzed_at.isoformat(),
+                    analysis.labeler,
+                    analysis.raw_response,
+                ),
+            )
+
+    def get_analysis(self, failure_id: str) -> FailureAnalysis | None:
+        row = self._connection.execute(
+            "SELECT * FROM failure_analyses WHERE failure_id = ?", (failure_id,)
+        ).fetchone()
+        if row is None:
+            return None
+        return FailureAnalysis.from_dict(
+            {
+                "failure_type": row["failure_type"],
+                "component": row["component"],
+                "severity": row["severity"],
+                "summary": row["summary"],
+                "analyzer": row["analyzer"],
+                "prompt_version": row["prompt_version"],
+                "analyzed_at": row["analyzed_at"],
+                "labeler": row["labeler"],
+                "raw_response": row["raw_response"],
+            }
+        )
+
+    def counts_by_type(self) -> dict[str, int]:
+        rows = self._connection.execute(
+            "SELECT failure_type, COUNT(*) AS n FROM failure_analyses GROUP BY failure_type"
+        ).fetchall()
+        return {row["failure_type"]: int(row["n"]) for row in rows}
+
+    def delete(self, failure_id: str) -> None:
+        with self._connection:
+            self._connection.execute("DELETE FROM failures WHERE failure_id = ?", (failure_id,))
+
+    def get(self, failure_id: str) -> Failure | None:
+        row = self._connection.execute(
+            "SELECT * FROM failures WHERE failure_id = ?", (failure_id.strip(),)
+        ).fetchone()
+        return self._build(row) if row is not None else None
+
+    def get_by_trace(self, trace_id: str) -> Failure | None:
+        row = self._connection.execute(
+            "SELECT * FROM failures WHERE trace_id = ?", (trace_id.strip(),)
+        ).fetchone()
+        return self._build(row) if row is not None else None
+
+    def list(
+        self, *, status: FailureStatus | None = None, limit: int = 50, offset: int = 0
+    ) -> list[FailureSummary]:
+        query = """
+            SELECT f.*, a.failure_type AS analysis_type, a.severity AS analysis_severity
+            FROM failures f
+            LEFT JOIN failure_analyses a ON a.failure_id = f.failure_id
+        """
+        parameters: list[Any] = []
+        if status is not None:
+            query += " WHERE f.status = ?"
+            parameters.append(status.value)
+        query += " ORDER BY f.detected_at, f.failure_id LIMIT ? OFFSET ?"
+        parameters += [limit, offset]
+
+        summaries: list[FailureSummary] = []
+        for row in self._connection.execute(query, parameters).fetchall():
+            signals = _load_signals(self._connection, row["failure_id"])
+            kinds: dict[str, None] = {}
+            for signal in signals:
+                kinds.setdefault(signal.kind.value, None)
+            summaries.append(
+                FailureSummary(
+                    failure_id=row["failure_id"],
+                    trace_id=row["trace_id"],
+                    status=FailureStatus(row["status"]),
+                    origin=FailureOrigin(row["origin"]),
+                    signals=len(signals),
+                    kinds=list(kinds),
+                    reviewer=row["reviewer"],
+                    failure_type=row["analysis_type"],
+                    severity=row["analysis_severity"],
+                )
+            )
+        return summaries
+
+    def count(self, *, status: FailureStatus | None = None) -> int:
+        if status is None:
+            row = self._connection.execute("SELECT COUNT(*) AS n FROM failures").fetchone()
+        else:
+            row = self._connection.execute(
+                "SELECT COUNT(*) AS n FROM failures WHERE status = ?", (status.value,)
+            ).fetchone()
+        return int(row["n"])
+
+    def counts_by_status(self) -> dict[FailureStatus, int]:
+        rows = self._connection.execute(
+            "SELECT status, COUNT(*) AS n FROM failures GROUP BY status"
+        ).fetchall()
+        return {FailureStatus(row["status"]): int(row["n"]) for row in rows}
+
+    def iter_all(self) -> Iterator[Failure]:
+        for row in self._connection.execute("SELECT * FROM failures ORDER BY failure_id"):
+            yield self._build(row)
+
+    def _build(self, row: sqlite3.Row) -> Failure:
+        return Failure(
+            failure_id=row["failure_id"],
+            trace_id=row["trace_id"],
+            status=FailureStatus(row["status"]),
+            origin=FailureOrigin(row["origin"]),
+            signals=_load_signals(self._connection, row["failure_id"]),
+            detected_at=datetime.fromisoformat(row["detected_at"]),
+            updated_at=datetime.fromisoformat(row["updated_at"]),
+            reviewer=row["reviewer"],
+            reason=row["reason"],
+        )
+
+
+def _load_signals(connection: sqlite3.Connection, failure_id: str) -> list[Signal]:
+    rows = connection.execute(
+        "SELECT * FROM failure_signals WHERE failure_id = ? ORDER BY position",
+        (failure_id,),
+    ).fetchall()
+    return [
+        Signal.from_dict(
+            {
+                "detector": row["detector"],
+                "kind": row["kind"],
+                "source": row["source"],
+                "summary": row["summary"],
+                "evidence": json.loads(row["evidence"]),
+            }
+        )
+        for row in rows
+    ]