evalkeep 0.1.0__py3-none-any.whl

evalkeep/storage/runs.py

@@ -0,0 +1,223 @@
+"""Persistence for evaluation runs and their results."""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+from datetime import datetime
+
+from evalkeep.runs import (
+    BaselinePromotion,
+    CaseResult,
+    CaseSummary,
+    ErrorKind,
+    EvaluationRun,
+    Outcome,
+    RunStatus,
+    summarize,
+)
+
+
+class AmbiguousRun(Exception):
+    """A run prefix matched more than one run."""
+
+
+class RunStore:
+    """Read/write access to the run tables."""
+
+    def __init__(self, connection: sqlite3.Connection) -> None:
+        self._connection = connection
+
+    def save(self, run: EvaluationRun, results: list[CaseResult]) -> None:
+        """Write a run and its results together, so neither exists alone."""
+        with self._connection:
+            self._connection.execute(
+                """
+                INSERT INTO evaluation_runs (
+                    run_id, target_id, suite_hash, tests, repetitions, status,
+                    runner, environment, started_at, finished_at, output_dir
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(run_id) DO UPDATE SET
+                    status = excluded.status,
+                    tests = excluded.tests,
+                    finished_at = excluded.finished_at,
+                    output_dir = excluded.output_dir
+                """,
+                (
+                    run.run_id,
+                    run.target_id,
+                    run.suite_hash,
+                    run.tests,
+                    run.repetitions,
+                    run.status.value,
+                    run.runner,
+                    json.dumps(run.environment, sort_keys=True),
+                    run.started_at.isoformat(),
+                    run.finished_at.isoformat() if run.finished_at else None,
+                    run.output_dir,
+                ),
+            )
+            self._connection.execute("DELETE FROM test_results WHERE run_id = ?", (run.run_id,))
+            self._connection.executemany(
+                """
+                INSERT INTO test_results (
+                    run_id, test_id, repetition, outcome, error_kind, error,
+                    latency_ms, observation, failed_assertions
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                [
+                    (
+                        run.run_id,
+                        result.test_id,
+                        result.repetition,
+                        result.outcome.value,
+                        result.error_kind.value if result.error_kind else None,
+                        result.error,
+                        result.latency_ms,
+                        result.observation,
+                        json.dumps(result.failed_assertions),
+                    )
+                    for result in results
+                ],
+            )
+
+    def get(self, run_id: str) -> EvaluationRun | None:
+        row = self._connection.execute(
+            "SELECT * FROM evaluation_runs WHERE run_id = ?", (run_id.strip(),)
+        ).fetchone()
+        return _build_run(row) if row is not None else None
+
+    def resolve(self, identifier: str) -> EvaluationRun | None:
+        """Find a run by its full ID or an unambiguous prefix.
+
+        Run IDs are 32 hex characters, which nobody types. Listings show a
+        prefix, so a prefix has to be usable -- an identifier a tool prints and
+        will not accept back is a bug, not a nicety.
+        """
+        cleaned = identifier.strip()
+        if not cleaned:
+            return None
+        exact = self.get(cleaned)
+        if exact is not None:
+            return exact
+
+        rows = self._connection.execute(
+            "SELECT * FROM evaluation_runs WHERE run_id LIKE ? || '%' ORDER BY run_id",
+            (cleaned,),
+        ).fetchall()
+        if len(rows) > 1:
+            matches = ", ".join(row["run_id"][:12] for row in rows)
+            raise AmbiguousRun(f"{cleaned!r} matches several runs: {matches}.")
+        return _build_run(rows[0]) if rows else None
+
+    def latest(self, target_id: str) -> EvaluationRun | None:
+        row = self._connection.execute(
+            """
+            SELECT * FROM evaluation_runs WHERE target_id = ?
+            ORDER BY started_at DESC LIMIT 1
+            """,
+            (target_id.strip(),),
+        ).fetchone()
+        return _build_run(row) if row is not None else None
+
+    def recent(self, *, limit: int = 20) -> list[EvaluationRun]:
+        return [
+            _build_run(row)
+            for row in self._connection.execute(
+                "SELECT * FROM evaluation_runs ORDER BY started_at DESC LIMIT ?", (limit,)
+            )
+        ]
+
+    def results(self, run_id: str) -> list[CaseResult]:
+        return [
+            _build_result(row)
+            for row in self._connection.execute(
+                "SELECT * FROM test_results WHERE run_id = ? ORDER BY test_id, repetition",
+                (run_id,),
+            )
+        ]
+
+    def promote(self, promotion: BaselinePromotion) -> None:
+        """Record that a run is now the baseline. Never inferred, always decided."""
+        with self._connection:
+            self._connection.execute(
+                """
+                INSERT INTO baseline_promotions (
+                    promotion_id, run_id, target_id, promoted_at, reviewer, reason
+                ) VALUES (?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    promotion.promotion_id,
+                    promotion.run_id,
+                    promotion.target_id,
+                    promotion.promoted_at.isoformat(),
+                    promotion.reviewer,
+                    promotion.reason,
+                ),
+            )
+
+    def current_baseline(self) -> BaselinePromotion | None:
+        row = self._connection.execute(
+            "SELECT * FROM baseline_promotions ORDER BY promoted_at DESC LIMIT 1"
+        ).fetchone()
+        return _build_promotion(row) if row is not None else None
+
+    def promotions(self, *, limit: int = 20) -> list[BaselinePromotion]:
+        return [
+            _build_promotion(row)
+            for row in self._connection.execute(
+                "SELECT * FROM baseline_promotions ORDER BY promoted_at DESC LIMIT ?",
+                (limit,),
+            )
+        ]
+
+    def summaries(self, run_id: str) -> dict[str, CaseSummary]:
+        """Every case in a run, with its repetitions taken together."""
+        return summarize(self.results(run_id))
+
+    def counts(self, run_id: str) -> dict[Outcome, int]:
+        rows = self._connection.execute(
+            "SELECT outcome, COUNT(*) AS n FROM test_results WHERE run_id = ? GROUP BY outcome",
+            (run_id,),
+        ).fetchall()
+        return {Outcome(row["outcome"]): int(row["n"]) for row in rows}
+
+
+def _build_promotion(row: sqlite3.Row) -> BaselinePromotion:
+    return BaselinePromotion(
+        promotion_id=row["promotion_id"],
+        run_id=row["run_id"],
+        target_id=row["target_id"],
+        reviewer=row["reviewer"],
+        reason=row["reason"],
+        promoted_at=datetime.fromisoformat(row["promoted_at"]),
+    )
+
+
+def _build_run(row: sqlite3.Row) -> EvaluationRun:
+    return EvaluationRun(
+        run_id=row["run_id"],
+        target_id=row["target_id"],
+        suite_hash=row["suite_hash"],
+        tests=row["tests"],
+        repetitions=row["repetitions"],
+        status=RunStatus(row["status"]),
+        runner=row["runner"],
+        environment=json.loads(row["environment"]),
+        started_at=datetime.fromisoformat(row["started_at"]),
+        finished_at=(datetime.fromisoformat(row["finished_at"]) if row["finished_at"] else None),
+        output_dir=row["output_dir"],
+    )
+
+
+def _build_result(row: sqlite3.Row) -> CaseResult:
+    return CaseResult(
+        test_id=row["test_id"],
+        outcome=Outcome(row["outcome"]),
+        error_kind=ErrorKind(row["error_kind"]) if row["error_kind"] else None,
+        error=row["error"],
+        latency_ms=row["latency_ms"],
+        observation=row["observation"],
+        failed_assertions=list(json.loads(row["failed_assertions"])),
+        repetition=row["repetition"],
+    )

evalkeep/storage/store.py

@@ -0,0 +1,429 @@
+"""The trace store: redacted traces in SQLite, never silently overwritten.
+
+The full redacted trace is stored as JSON on ``traces`` and is the source of
+truth. Events are also written as rows, as a derived index so that detection can
+ask "which traces called ``refund_order``" without deserializing every trace;
+both are written in the same transaction, so the index cannot drift.
+
+Storing a trace ID that already exists is never an overwrite. It is either a
+no-op (the stored trace has the same content, so re-ingesting a file is safe) or
+a conflict the caller must resolve (same ID, different content).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import sqlite3
+from collections.abc import Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from enum import StrEnum
+from pathlib import Path
+from typing import Any
+
+from evalkeep.errors import CommandError
+from evalkeep.hashing import content_hash
+from evalkeep.redaction import RedactionSummary
+from evalkeep.storage.clusters import ClusterStore
+from evalkeep.storage.failures import FailureStore
+from evalkeep.storage.migrations import apply_migrations
+from evalkeep.storage.regression import RegressionStore
+from evalkeep.storage.runs import RunStore
+from evalkeep.trace import NormalizedTrace, ToolCallEvent, ToolResultEvent
+
+
+class StoreResult(StrEnum):
+    """What happened -- or, for a dry run, what would have happened."""
+
+    STORED = "stored"
+    #: Same trace_id, same content: already ingested, nothing to do.
+    ALREADY_STORED = "already_stored"
+    #: Same trace_id, different content: refused, the caller must decide.
+    ID_CONFLICT = "id_conflict"
+    #: Different trace_id, same content: the interaction is already covered.
+    CONTENT_DUPLICATE = "content_duplicate"
+
+
+@dataclass(frozen=True)
+class StoreOutcome:
+    result: StoreResult
+    trace_id: str
+    content_hash: str
+    #: For a content duplicate, the trace already holding this content.
+    existing_trace_id: str | None = None
+
+    @property
+    def written(self) -> bool:
+        return self.result is StoreResult.STORED
+
+
+@dataclass(frozen=True)
+class TraceOccurrence:
+    """One sighting of an interaction.
+
+    Distinct from a trace: `traces` holds one row per interaction, and this
+    holds one row per time that interaction was seen. The same bug hitting two
+    hundred users is one trace and two hundred occurrences.
+    """
+
+    occurrence_id: str
+    canonical_trace_id: str
+    content_hash: str
+    #: The ID this sighting carried, which may differ from the canonical one.
+    trace_id: str
+    source: str | None = None
+    agent: str | None = None
+    model: str | None = None
+    recorded_at: str | None = None
+    ingested_at: str = ""
+
+
+@dataclass(frozen=True)
+class OccurrenceStats:
+    """How often an interaction was seen, and across what."""
+
+    count: int = 0
+    first_seen: str | None = None
+    last_seen: str | None = None
+    #: Distinct agent versions this interaction was observed on.
+    agents: tuple[str, ...] = ()
+
+    @property
+    def recurring(self) -> bool:
+        return self.count > 1
+
+
+@dataclass(frozen=True)
+class StoredTrace:
+    """A trace as it came back out of the database."""
+
+    trace: NormalizedTrace
+    content_hash: str
+    ingested_at: str
+    redactions: int
+    redaction_summary: dict[str, int]
+    occurrences: OccurrenceStats = field(default_factory=OccurrenceStats)
+
+
+@dataclass(frozen=True)
+class TraceSummary:
+    """One row of ``evalkeep trace list``."""
+
+    trace_id: str
+    status: str
+    source: str | None
+    recorded_at: str | None
+    events: int
+    redactions: int
+    occurrences: int = 1
+
+
+class TraceStore:
+    """Read/write access to one project's trace database."""
+
+    def __init__(self, connection: sqlite3.Connection) -> None:
+        self._connection = connection
+
+    @classmethod
+    @contextmanager
+    def open(cls, path: Path) -> Iterator[TraceStore]:
+        """Open (creating and migrating as needed) and always close cleanly."""
+        try:
+            path.parent.mkdir(parents=True, exist_ok=True)
+            connection = sqlite3.connect(path)
+        except (OSError, sqlite3.Error) as exc:
+            raise CommandError(f"Could not open the database {path}: {exc}") from exc
+
+        connection.row_factory = sqlite3.Row
+        try:
+            connection.execute("PRAGMA foreign_keys = ON")
+            apply_migrations(connection)
+            yield cls(connection)
+        finally:
+            connection.close()
+
+    @property
+    def clusters(self) -> ClusterStore:
+        """Clusterings, sharing this store's connection."""
+        return ClusterStore(self._connection)
+
+    @property
+    def runs(self) -> RunStore:
+        """Evaluation runs, sharing this store's connection."""
+        return RunStore(self._connection)
+
+    @property
+    def tests(self) -> RegressionStore:
+        """Regression-test drafts, sharing this store's connection."""
+        return RegressionStore(self._connection)
+
+    @property
+    def failures(self) -> FailureStore:
+        """Failure candidates, sharing this store's connection."""
+        return FailureStore(self._connection)
+
+    # -- writing ---------------------------------------------------------
+
+    def add(
+        self, trace: NormalizedTrace, *, redaction: RedactionSummary | None = None
+    ) -> StoreOutcome:
+        """Store a redacted trace, or explain why it was not stored."""
+        outcome = self.classify(trace)
+        if not outcome.written:
+            return outcome
+
+        summary = redaction or RedactionSummary()
+        payload = trace.model_dump(mode="json")
+        now = datetime.now(UTC).isoformat()
+        try:
+            with self._connection:  # one transaction: trace and its events
+                self._connection.execute(
+                    """
+                    INSERT INTO traces (
+                        trace_id, content_hash, schema_version, status, source,
+                        recorded_at, ingested_at, redactions, redaction_summary, payload
+                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                    """,
+                    (
+                        trace.trace_id,
+                        outcome.content_hash,
+                        trace.schema_version,
+                        trace.outcome.status.value,
+                        trace.metadata.source,
+                        _isoformat(trace.metadata.recorded_at),
+                        now,
+                        summary.total,
+                        json.dumps(summary.to_dict(), sort_keys=True),
+                        json.dumps(payload, sort_keys=True),
+                    ),
+                )
+                self._connection.executemany(
+                    """
+                    INSERT INTO events (
+                        trace_id, position, event_id, type, tool, call_id, timestamp, payload
+                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+                    """,
+                    list(_event_rows(trace, payload)),
+                )
+        except sqlite3.Error as exc:  # pragma: no cover - defensive
+            raise CommandError(f"Could not store trace {trace.trace_id!r}: {exc}") from exc
+        return outcome
+
+    def record_occurrence(
+        self, trace: NormalizedTrace, *, canonical_trace_id: str, digest: str
+    ) -> bool:
+        """Note that this interaction was seen. Returns True if it was new.
+
+        The occurrence ID is derived from the sighting itself, so re-ingesting a
+        file is a no-op rather than a way to inflate a frequency count.
+        """
+        occurrence_id = occurrence_id_for(digest, trace)
+        with self._connection:
+            cursor = self._connection.execute(
+                """
+                INSERT OR IGNORE INTO trace_occurrences (
+                    occurrence_id, canonical_trace_id, content_hash, trace_id,
+                    source, agent, model, recorded_at, ingested_at
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    occurrence_id,
+                    canonical_trace_id,
+                    digest,
+                    trace.trace_id,
+                    trace.metadata.source,
+                    trace.metadata.agent,
+                    trace.metadata.model,
+                    _isoformat(trace.metadata.recorded_at),
+                    datetime.now(UTC).isoformat(),
+                ),
+            )
+            return cursor.rowcount > 0
+
+    def occurrences(self, trace_id: str) -> OccurrenceStats:
+        """How often this interaction was seen, and on which agent versions."""
+        row = self._connection.execute(
+            """
+            SELECT COUNT(*) AS n,
+                   MIN(COALESCE(recorded_at, ingested_at)) AS first_seen,
+                   MAX(COALESCE(recorded_at, ingested_at)) AS last_seen
+            FROM trace_occurrences WHERE canonical_trace_id = ?
+            """,
+            (trace_id.strip(),),
+        ).fetchone()
+        agents = [
+            r["agent"]
+            for r in self._connection.execute(
+                """
+                SELECT DISTINCT agent FROM trace_occurrences
+                WHERE canonical_trace_id = ? AND agent IS NOT NULL
+                ORDER BY agent
+                """,
+                (trace_id.strip(),),
+            )
+        ]
+        return OccurrenceStats(
+            count=int(row["n"]),
+            first_seen=row["first_seen"],
+            last_seen=row["last_seen"],
+            agents=tuple(agents),
+        )
+
+    def occurrence_list(self, trace_id: str) -> list[TraceOccurrence]:
+        return [
+            TraceOccurrence(
+                occurrence_id=row["occurrence_id"],
+                canonical_trace_id=row["canonical_trace_id"],
+                content_hash=row["content_hash"],
+                trace_id=row["trace_id"],
+                source=row["source"],
+                agent=row["agent"],
+                model=row["model"],
+                recorded_at=row["recorded_at"],
+                ingested_at=row["ingested_at"],
+            )
+            for row in self._connection.execute(
+                """
+                SELECT * FROM trace_occurrences WHERE canonical_trace_id = ?
+                ORDER BY COALESCE(recorded_at, ingested_at), occurrence_id
+                """,
+                (trace_id.strip(),),
+            )
+        ]
+
+    def classify(self, trace: NormalizedTrace) -> StoreOutcome:
+        """Decide what storing ``trace`` would do, without writing anything."""
+        digest = content_hash(trace)
+
+        existing = self._connection.execute(
+            "SELECT content_hash FROM traces WHERE trace_id = ?", (trace.trace_id,)
+        ).fetchone()
+        if existing is not None:
+            result = (
+                StoreResult.ALREADY_STORED
+                if existing["content_hash"] == digest
+                else StoreResult.ID_CONFLICT
+            )
+            return StoreOutcome(result, trace.trace_id, digest)
+
+        same_content = self._connection.execute(
+            "SELECT trace_id FROM traces WHERE content_hash = ? ORDER BY trace_id LIMIT 1",
+            (digest,),
+        ).fetchone()
+        if same_content is not None:
+            return StoreOutcome(
+                StoreResult.CONTENT_DUPLICATE,
+                trace.trace_id,
+                digest,
+                existing_trace_id=same_content["trace_id"],
+            )
+
+        return StoreOutcome(StoreResult.STORED, trace.trace_id, digest)
+
+    # -- reading ---------------------------------------------------------
+
+    def get(self, trace_id: str) -> StoredTrace | None:
+        row = self._connection.execute(
+            "SELECT * FROM traces WHERE trace_id = ?", (trace_id.strip(),)
+        ).fetchone()
+        if row is None:
+            return None
+        return StoredTrace(
+            trace=NormalizedTrace.model_validate_json(row["payload"]),
+            content_hash=row["content_hash"],
+            ingested_at=row["ingested_at"],
+            redactions=row["redactions"],
+            redaction_summary=json.loads(row["redaction_summary"]),
+            occurrences=self.occurrences(row["trace_id"]),
+        )
+
+    def list(
+        self, *, limit: int = 50, offset: int = 0, status: str | None = None
+    ) -> list[TraceSummary]:
+        query = """
+            SELECT t.trace_id, t.status, t.source, t.recorded_at, t.redactions,
+                   (SELECT COUNT(*) FROM events e WHERE e.trace_id = t.trace_id) AS events,
+                   (SELECT COUNT(*) FROM trace_occurrences o
+                    WHERE o.canonical_trace_id = t.trace_id) AS occurrences
+            FROM traces t
+        """
+        parameters: list[Any] = []
+        if status is not None:
+            query += " WHERE t.status = ?"
+            parameters.append(status)
+        query += " ORDER BY t.ingested_at, t.trace_id LIMIT ? OFFSET ?"
+        parameters += [limit, offset]
+
+        return [
+            TraceSummary(
+                trace_id=row["trace_id"],
+                status=row["status"],
+                source=row["source"],
+                recorded_at=row["recorded_at"],
+                events=row["events"],
+                redactions=row["redactions"],
+                occurrences=row["occurrences"],
+            )
+            for row in self._connection.execute(query, parameters)
+        ]
+
+    def iter_traces(self) -> Iterator[NormalizedTrace]:
+        """Stream every stored trace in ingest order, without loading them all."""
+        cursor = self._connection.execute(
+            "SELECT payload FROM traces ORDER BY ingested_at, trace_id"
+        )
+        for row in cursor:
+            yield NormalizedTrace.model_validate_json(row["payload"])
+
+    def count(self, *, status: str | None = None) -> int:
+        if status is None:
+            row = self._connection.execute("SELECT COUNT(*) AS n FROM traces").fetchone()
+        else:
+            row = self._connection.execute(
+                "SELECT COUNT(*) AS n FROM traces WHERE status = ?", (status,)
+            ).fetchone()
+        return int(row["n"])
+
+    def event_count(self) -> int:
+        return int(self._connection.execute("SELECT COUNT(*) AS n FROM events").fetchone()["n"])
+
+
+def occurrence_id_for(digest: str, trace: NormalizedTrace) -> str:
+    """Identify a sighting by what distinguishes it from other sightings.
+
+    Two records of the same interaction, carrying the same ID and the same
+    recorded time, are the same sighting seen twice -- most often because a file
+    was ingested again -- and must not count twice.
+    """
+    material = "\n".join(
+        [
+            digest,
+            trace.trace_id,
+            _isoformat(trace.metadata.recorded_at) or "",
+            trace.metadata.source or "",
+        ]
+    )
+    return hashlib.sha256(material.encode("utf-8")).hexdigest()[:24]
+
+
+def _event_rows(trace: NormalizedTrace, payload: dict[str, Any]) -> Iterator[tuple[Any, ...]]:
+    """Flatten events into index rows, keeping their recorded order."""
+    for position, (event, dumped) in enumerate(zip(trace.events, payload["events"], strict=True)):
+        tool = event.tool if isinstance(event, ToolCallEvent | ToolResultEvent) else None
+        call_id = event.call_id if isinstance(event, ToolCallEvent | ToolResultEvent) else None
+        yield (
+            trace.trace_id,
+            position,
+            event.event_id,
+            event.type,
+            tool,
+            call_id,
+            _isoformat(event.timestamp),
+            json.dumps(dumped, sort_keys=True),
+        )
+
+
+def _isoformat(value: datetime | None) -> str | None:
+    return value.isoformat() if value is not None else None