evalkeep 0.1.0__py3-none-any.whl

evalkeep/ingest.py

@@ -0,0 +1,257 @@
+"""The ingest pipeline: validate, redact, hash, store -- in that order.
+
+Redaction happens between validation and storage, in memory, so a raw value
+never reaches the database. The pipeline streams: issues are written to the
+error JSONL as they are found and only a bounded sample is kept for display, so
+validating 100k traces costs the set of trace IDs seen so far, not the file.
+
+Three modes share one implementation:
+
+* **validate** (no store) -- parse and check only. Needs no project.
+* **dry run** -- redact and ask the store what *would* happen, writing nothing.
+* **ingest** -- the same, then commit.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterator
+from contextlib import ExitStack
+from dataclasses import dataclass, field
+from enum import StrEnum
+from pathlib import Path
+from typing import TextIO
+
+from evalkeep.adapters import AdapterRecord, IssueKind, TraceAdapter, TraceIssue
+from evalkeep.errors import CommandError, ExitCode
+from evalkeep.redaction import RedactionSummary, Redactor, risky_identifiers
+from evalkeep.storage import StoreOutcome, StoreResult, TraceStore
+
+#: Issues kept for terminal display; the rest go to the error JSONL.
+DEFAULT_SAMPLE_LIMIT = 20
+
+
+class IngestMode(StrEnum):
+    VALIDATE = "validate"
+    DRY_RUN = "dry-run"
+    STORE = "ingest"
+
+
+@dataclass
+class IngestReport:
+    """What a pass over a trace file found. Counts are exact; ``sample`` is bounded."""
+
+    path: Path
+    adapter: str
+    mode: IngestMode = IngestMode.VALIDATE
+
+    # Validation
+    records: int = 0
+    valid: int = 0
+    invalid: int = 0
+    issue_count: int = 0
+    duplicate_ids: int = 0
+
+    # Storage
+    stored: int = 0
+    already_stored: int = 0
+    content_duplicates: int = 0
+    id_conflicts: int = 0
+    #: New sightings recorded. An interaction seen again is not stored twice,
+    #: but the fact that it happened again is kept.
+    occurrences: int = 0
+
+    # Redaction
+    redactions: int = 0
+    redacted_traces: int = 0
+    #: Traces whose identifiers look like they carry personal data, counted
+    #: only when pseudonymization is off and they were therefore stored as-is.
+    identifier_risks: int = 0
+    redaction_summary: RedactionSummary = field(default_factory=RedactionSummary)
+
+    error_path: Path | None = None
+    sample: list[TraceIssue] = field(default_factory=list)
+    #: Things worth telling the user that are not record errors.
+    notices: list[str] = field(default_factory=list)
+
+    @property
+    def ok(self) -> bool:
+        """Records that could not be handled at all -- duplicates are not failures."""
+        return self.invalid == 0 and self.id_conflicts == 0
+
+    @property
+    def truncated(self) -> int:
+        return max(0, self.issue_count - len(self.sample))
+
+    @property
+    def skipped(self) -> int:
+        """Valid traces the store already knew about."""
+        return self.already_stored + self.content_duplicates
+
+    @property
+    def exit_code(self) -> ExitCode:
+        return ExitCode.OK if self.ok else ExitCode.RECORD_ERRORS
+
+
+def iter_records(path: Path, adapter: TraceAdapter) -> Iterator[AdapterRecord]:
+    """Adapter records plus the duplicate-ID check the adapter cannot make.
+
+    An adapter sees one record at a time and so cannot know that a trace ID has
+    already appeared in this file. The second occurrence is rejected rather than
+    merged: accepting it would decide, in the wrong place, which copy wins.
+    """
+    seen: set[str] = set()
+    for record in adapter.read(path):
+        if record.trace is None:
+            yield record
+            continue
+        trace_id = record.trace.trace_id
+        if trace_id in seen:
+            yield AdapterRecord.rejected(
+                record.line,
+                TraceIssue(
+                    line=record.line,
+                    kind=IssueKind.DUPLICATE_ID,
+                    message=f"trace_id {trace_id!r} already appeared earlier in this file",
+                    trace_id=trace_id,
+                    hint="Trace IDs must be unique; the first occurrence is kept.",
+                ),
+            )
+            continue
+        seen.add(trace_id)
+        yield record
+
+
+def ingest_file(
+    path: Path,
+    adapter: TraceAdapter,
+    *,
+    store: TraceStore | None = None,
+    redactor: Redactor | None = None,
+    dry_run: bool = False,
+    error_path: Path | None = None,
+    sample_limit: int = DEFAULT_SAMPLE_LIMIT,
+) -> IngestReport:
+    """Run the pipeline over ``path``. Without ``store``, validation only."""
+    _check_readable_file(path)
+    mode = (
+        IngestMode.VALIDATE
+        if store is None
+        else (IngestMode.DRY_RUN if dry_run else IngestMode.STORE)
+    )
+    report = IngestReport(path=path, adapter=adapter.name, mode=mode, error_path=error_path)
+    redactor = redactor or Redactor()
+
+    with ExitStack() as stack:
+        errors: TextIO | None = None
+        if error_path is not None:
+            errors = stack.enter_context(_open_error_file(error_path))
+
+        def report_issue(issue: TraceIssue) -> None:
+            report.issue_count += 1
+            if issue.kind is IssueKind.DUPLICATE_ID:
+                report.duplicate_ids += 1
+            if len(report.sample) < sample_limit:
+                report.sample.append(issue)
+            if errors is not None:
+                errors.write(json.dumps(issue.to_dict(), sort_keys=True) + "\n")
+
+        for record in iter_records(path, adapter):
+            report.records += 1
+            if record.trace is None:
+                report.invalid += 1
+                for issue in record.issues:
+                    report_issue(issue)
+                continue
+
+            report.valid += 1
+            if store is None:
+                continue
+
+            # Redaction sits here on purpose: between a trace being valid and it
+            # touching the database, with no path around it.
+            if not redactor.pseudonymizing:
+                risks = risky_identifiers(record.trace)
+                if risks:
+                    report.identifier_risks += 1
+                    for risk in risks:
+                        if risk not in report.notices and len(report.notices) < 5:
+                            report.notices.append(risk)
+
+            redacted, summary = redactor.redact(record.trace)
+            report.redactions += summary.total
+            report.redacted_traces += 1 if summary.total else 0
+            report.redaction_summary.merge(summary)
+
+            outcome = (
+                store.classify(redacted) if dry_run else store.add(redacted, redaction=summary)
+            )
+            _count_outcome(report, outcome.result)
+
+            # Every accepted sighting is recorded, whether or not the
+            # interaction itself was new. Deduplication belongs to the test
+            # suite; the evidence keeps its count.
+            canonical = _canonical_id(outcome)
+            if (
+                not dry_run
+                and canonical is not None
+                and store.record_occurrence(
+                    redacted, canonical_trace_id=canonical, digest=outcome.content_hash
+                )
+            ):
+                report.occurrences += 1
+            if outcome.result is StoreResult.ID_CONFLICT:
+                report_issue(
+                    TraceIssue(
+                        line=record.line,
+                        kind=IssueKind.ID_CONFLICT,
+                        message=(
+                            f"trace_id {redacted.trace_id!r} is already stored with "
+                            "different content"
+                        ),
+                        trace_id=redacted.trace_id,
+                        hint="Give the new trace a different ID, or remove the stored one.",
+                    )
+                )
+
+    return report
+
+
+def _canonical_id(outcome: StoreOutcome) -> str | None:
+    """Which stored trace this sighting belongs to, or None if it was refused."""
+    match outcome.result:
+        case StoreResult.STORED | StoreResult.ALREADY_STORED:
+            return outcome.trace_id
+        case StoreResult.CONTENT_DUPLICATE:
+            return outcome.existing_trace_id
+        case _:
+            return None
+
+
+def _count_outcome(report: IngestReport, result: StoreResult) -> None:
+    match result:
+        case StoreResult.STORED:
+            report.stored += 1
+        case StoreResult.ALREADY_STORED:
+            report.already_stored += 1
+        case StoreResult.CONTENT_DUPLICATE:
+            report.content_duplicates += 1
+        case StoreResult.ID_CONFLICT:
+            report.id_conflicts += 1
+
+
+def _open_error_file(error_path: Path) -> TextIO:
+    try:
+        error_path.parent.mkdir(parents=True, exist_ok=True)
+        return error_path.open("w", encoding="utf-8")
+    except OSError as exc:
+        raise CommandError(f"Could not write the error file {error_path}: {exc}") from exc
+
+
+def _check_readable_file(path: Path) -> None:
+    if not path.exists():
+        raise CommandError(f"{path} does not exist.")
+    if path.is_dir():
+        raise CommandError(f"{path} is a directory, not a trace file.")
+    if not path.is_file():
+        raise CommandError(f"{path} is not a regular file.")

evalkeep/prompts.py

@@ -0,0 +1,127 @@
+"""Versioned prompts and the strict JSON schema their answers must satisfy.
+
+The version is part of every cache key and is stored with every analysis, so a
+prompt change never silently mixes old and new labels in one dataset. Editing
+the prompt text without bumping :data:`FAILURE_ANALYSIS_PROMPT_VERSION` is a
+bug: it would serve stale cached answers for a question you no longer ask.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from evalkeep.analysis import Component, FailureType, Severity
+from evalkeep.detectors import Signal
+from evalkeep.trace import (
+    EvaluationEvent,
+    MessageEvent,
+    NormalizedTrace,
+    ToolCallEvent,
+    ToolResultEvent,
+)
+
+FAILURE_ANALYSIS_PROMPT_VERSION = 1
+
+MAX_SUMMARY_LENGTH = 240
+
+FAILURE_ANALYSIS_SCHEMA: dict[str, Any] = {
+    "type": "object",
+    "properties": {
+        "failure_type": {
+            "type": "string",
+            "enum": [member.value for member in FailureType],
+            "description": "The kind of failure, from the fixed list.",
+        },
+        "component": {
+            "type": "string",
+            "enum": [member.value for member in Component],
+            "description": "Where in the agent the failure originated.",
+        },
+        "severity": {
+            "type": "string",
+            "enum": [member.value for member in Severity],
+            "description": "How much damage this failure does to a user.",
+        },
+        "summary": {
+            "type": "string",
+            "description": (
+                "One sentence naming the specific mistake, in terms that would "
+                "match other traces with the same underlying problem."
+            ),
+        },
+    },
+    "required": ["failure_type", "component", "severity", "summary"],
+    "additionalProperties": False,
+}
+
+FAILURE_ANALYSIS_SYSTEM = """\
+You classify failures in recorded AI-agent interactions so that similar failures \
+can be grouped together.
+
+Rules:
+- Describe only what the recorded interaction shows. Do not speculate about code \
+you cannot see, and do not propose fixes.
+- Write the summary so that two traces with the same underlying problem would \
+receive near-identical summaries. Name the mistake, not the customer, the order \
+or the wording of this particular request.
+- Values shown as [REDACTED:...] were removed before you saw them. Treat them as \
+opaque; never guess what they were.
+- Choose the most specific failure_type that fits. Use "other" only when nothing \
+else applies.
+- Severity is about user impact: critical means money, data or safety; low means \
+cosmetic or easily noticed.\
+"""
+
+
+def failure_analysis_prompt(trace: NormalizedTrace, signals: list[Signal]) -> str:
+    """The user-turn prompt describing one failing interaction."""
+    sections = [
+        "Here is a recorded interaction that has been marked as a failure.",
+        "",
+        "## Evidence that it failed",
+    ]
+    sections.extend(f"- ({signal.kind.value}) {signal.summary}" for signal in signals)
+    sections += ["", "## The interaction", _render_trace(trace)]
+    sections += [
+        "",
+        "Classify this failure. Respond with a JSON object matching the required schema.",
+    ]
+    return "\n".join(sections)
+
+
+def _render_trace(trace: NormalizedTrace) -> str:
+    """A compact, stable rendering. Stable matters: it feeds the cache key."""
+    lines: list[str] = []
+    if trace.input.text:
+        lines.append(f"user: {trace.input.text}")
+    for message in trace.input.messages:
+        lines.append(f"{message.role.value}: {message.content}")
+
+    for event in trace.events:
+        if isinstance(event, ToolCallEvent):
+            arguments = json.dumps(event.arguments, sort_keys=True)
+            lines.append(f"tool_call: {event.tool}({arguments})")
+        elif isinstance(event, ToolResultEvent):
+            result = json.dumps(event.result, sort_keys=True, default=str)
+            suffix = f" error={event.error}" if event.error else ""
+            lines.append(f"tool_result: {event.tool} -> {result}{suffix}")
+        elif isinstance(event, MessageEvent):
+            lines.append(f"{event.role.value}: {event.content}")
+        elif isinstance(event, EvaluationEvent):
+            verdict = {True: "pass", False: "fail", None: "unrecorded"}[event.passed]
+            lines.append(f"evaluation: {event.name} {verdict}")
+
+    if trace.output is not None and trace.output.text:
+        lines.append(f"assistant: {trace.output.text}")
+    for message in trace.output.messages if trace.output else []:
+        lines.append(f"{message.role.value}: {message.content}")
+
+    if trace.outcome.feedback is not None and trace.outcome.feedback.comment:
+        lines.append(f"feedback: {trace.outcome.feedback.comment}")
+    for evaluation in trace.outcome.evaluations:
+        if evaluation.passed is False:
+            reason = f": {evaluation.reason}" if evaluation.reason else ""
+            lines.append(f"failed evaluation: {evaluation.name}{reason}")
+
+    return "\n".join(lines)

evalkeep/pseudonyms.py

@@ -0,0 +1,82 @@
+"""Deterministic pseudonyms for identifiers that may carry customer data.
+
+Redaction deliberately leaves identifiers alone, because rewriting them would
+break the links the pipeline runs on. That is fine when a `trace_id` is a UUID
+and dangerous when it is `order-jane@example.com-2026-06-01`.
+
+Pseudonymization resolves the tension instead of trading one problem for the
+other. Each identifier becomes a token derived from a per-project secret salt:
+
+* **Deterministic**, so the same original always yields the same token and every
+  link in the pipeline survives.
+* **Not reversible from the database**, because the original is never stored --
+  only the token is.
+* **Still usable by hand**, because a lookup can hash whatever the user typed
+  and search for that. You keep using the IDs your own systems know.
+* **Scoped to one project**, because the salt is per-project and never
+  committed. Two projects produce different tokens for the same original, so a
+  shared export leaks nothing about another project's data.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import secrets
+from pathlib import Path
+
+from evalkeep.errors import CommandError
+
+SALT_FILENAME = "salt"
+SALT_BYTES = 32
+TOKEN_LENGTH = 12
+
+#: Which identifier gets which readable prefix, so a pseudonym still looks like
+#: the kind of thing it replaced.
+PREFIXES: dict[str, str] = {
+    "trace_id": "trace",
+    "event_id": "event",
+    "call_id": "call",
+}
+
+
+class Pseudonymizer:
+    """Turns an identifier into a stable token, given a project's salt."""
+
+    def __init__(self, salt: bytes) -> None:
+        if len(salt) < 16:
+            raise ValueError("the salt must be at least 16 bytes")
+        self._salt = salt
+
+    def token(self, value: str, *, field: str) -> str:
+        """A stable pseudonym for ``value``, prefixed by the kind of ID it is."""
+        digest = hashlib.blake2b(
+            value.encode("utf-8"), digest_size=TOKEN_LENGTH // 2, key=self._salt
+        ).hexdigest()
+        return f"{PREFIXES.get(field, 'id')}-{digest}"
+
+    @classmethod
+    def load(cls, path: Path) -> Pseudonymizer:
+        """Read a project's salt, creating one on first use."""
+        if not path.is_file():
+            return cls(_create_salt(path))
+        try:
+            salt = bytes.fromhex(path.read_text(encoding="utf-8").strip())
+        except (OSError, ValueError) as exc:
+            raise CommandError(
+                f"Could not read the pseudonymization salt at {path}: {exc}.",
+                hint="If it was lost, previously stored identifiers cannot be "
+                "matched again; re-ingest into a fresh project.",
+            ) from exc
+        return cls(salt)
+
+
+def _create_salt(path: Path) -> bytes:
+    salt = secrets.token_bytes(SALT_BYTES)
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(salt.hex(), encoding="utf-8")
+        # The salt is what makes the tokens unguessable; treat it like a key.
+        path.chmod(0o600)
+    except OSError as exc:
+        raise CommandError(f"Could not write the salt to {path}: {exc}") from exc
+    return salt