evalkeep 0.1.0__py3-none-any.whl

evalkeep/exporters/promptfoo.py

@@ -0,0 +1,219 @@
+"""Generating a Promptfoo configuration from approved tests.
+
+Evalkeep does not execute agents; it hands an established runner a suite and
+reads the results back. That division is the point of the project, so this
+module's only job is translation -- and translation has two rules:
+
+* **Every value is embedded as JSON, never interpolated as text.** Expectations
+  contain data that came from a trace: order IDs, output fragments, tool names.
+  Pasting those into a JavaScript expression by concatenation would be an
+  injection bug with the same shape as SQL injection, so every literal goes
+  through ``json.dumps``.
+* **Every provider kind is adapted to one response shape.** Assertions are
+  written once, against ``{text, toolCalls}``, and the provider configuration is
+  what differs between an HTTP endpoint and a local script.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import Any
+
+from evalkeep.regression import Expectation, ExpectationType, RegressionTest
+from evalkeep.targets import Target, TargetKind
+
+#: Reads a dotted path out of a tool call's arguments, tolerating gaps.
+_ARGUMENT_GETTER = "(p,o)=>p.split('.').reduce((a,k)=>a==null?a:a[k],o)"
+#: Parenthesised deliberately. Without the outer parentheses, `.some()` binds
+#: tighter than `||`, so `(output&&output.toolCalls)||[].some(...)` evaluates to
+#: the tool-call array itself -- truthy -- and every tool assertion passes no
+#: matter what the agent did.
+_CALLS = "((output&&output.toolCalls)||[])"
+_TEXT = "String((output&&output.text)??'')"
+
+
+def build_config(
+    tests: list[RegressionTest],
+    target: Target,
+    *,
+    description: str | None = None,
+    project_root: Path | None = None,
+    config_dir: Path | None = None,
+) -> dict[str, Any]:
+    """A complete promptfooconfig document for one target and one suite."""
+    return {
+        "description": description or f"evalkeep suite against {target.target_id}",
+        "providers": [provider_for(target, project_root=project_root, config_dir=config_dir)],
+        # The prompt is the test input verbatim: Evalkeep tests an application,
+        # not a prompt template.
+        "prompts": ["{{input}}"],
+        "tests": [build_test_case(test) for test in tests],
+    }
+
+
+def replay_warnings(tests: list[RegressionTest], target: Target) -> list[str]:
+    """Cases where fixtures were recorded but cannot reach the target.
+
+    Silence here would be the worst outcome: the run would look like a faithful
+    replay while actually re-querying live tools, and the comparison would be
+    answering a different question than the one it reports.
+    """
+    with_fixtures = [test for test in tests if test.fixtures]
+    if not with_fixtures:
+        return []
+
+    if target.kind is TargetKind.HTTP:
+        body = json.dumps(target.body)
+        if FIXTURES_VAR not in body:
+            return [
+                f"{len(with_fixtures)} test(s) carry recorded fixtures, but the "
+                f"body of target {target.target_id!r} never references "
+                f"{{{{{FIXTURES_VAR}}}}}, so the agent will call its real tools "
+                "instead of replaying what was recorded."
+            ]
+    elif target.kind is TargetKind.MODEL:
+        return [
+            f"{len(with_fixtures)} test(s) carry recorded fixtures, which a direct "
+            "model provider cannot receive. The model will be asked to choose "
+            "tools without the recorded results."
+        ]
+    return []
+
+
+def provider_for(
+    target: Target,
+    *,
+    project_root: Path | None = None,
+    config_dir: Path | None = None,
+) -> dict[str, Any]:
+    """Translate a target into a Promptfoo provider.
+
+    Script paths in a target are written relative to the project, because that
+    is where a person reading ``targets.yaml`` expects them to be. The runner
+    resolves them relative to the generated config instead, and the config is a
+    build artifact that can land anywhere -- so the path is rewritten here to
+    point at the same file from wherever the config is being written.
+    """
+    match target.kind:
+        case TargetKind.HTTP:
+            return {
+                "id": target.url,
+                "config": {
+                    "method": target.method,
+                    "headers": dict(target.headers),
+                    "body": target.body,
+                    # Normalize whatever the service returns into the one shape
+                    # the assertions are written against.
+                    "transformResponse": (
+                        "({text: String("
+                        f"{target.extract.output} ?? ''), toolCalls: "
+                        f"{target.extract.tool_calls or '[]'} ?? []}})"
+                    ),
+                },
+            }
+        case TargetKind.PYTHON:
+            suffix = f":{target.function}" if target.function else ""
+            script = _script_path(target.path, project_root, config_dir)
+            return {"id": f"file://{script}{suffix}"}
+        case TargetKind.JAVASCRIPT:
+            script = _script_path(target.path, project_root, config_dir)
+            return {"id": f"file://{script}"}
+        case _:
+            return {"id": target.provider}
+
+
+#: The variable recorded tool results are published under. A target that wants
+#: reproducible replay reads this; one that ignores it behaves as before.
+FIXTURES_VAR = "fixtures"
+
+
+def build_test_case(test: RegressionTest) -> dict[str, Any]:
+    """One Promptfoo test case, described by its stable test ID.
+
+    Recorded fixtures are published as a test variable rather than injected.
+    Evalkeep cannot intercept a black-box agent's tool calls -- it can only put
+    the recorded results where the target can reach them, and say so. What the
+    target does with them is the target's decision, and an honest boundary.
+    """
+    variables: dict[str, Any] = {"input": _input_text(test)}
+    if test.fixtures:
+        variables[FIXTURES_VAR] = [fixture.for_replay() for fixture in test.fixtures]
+    return {
+        "description": test.test_id,
+        "vars": variables,
+        "assert": [assertion(expectation) for expectation in test.expectations],
+        "metadata": {
+            "test_id": test.test_id,
+            "trace_id": test.provenance.trace_id,
+            "failure_type": test.provenance.failure_type,
+            "severity": test.provenance.severity,
+        },
+    }
+
+
+def assertion(expectation: Expectation) -> dict[str, Any]:
+    """Translate one expectation into a Promptfoo assertion."""
+    if expectation.type is ExpectationType.HUMAN_RUBRIC:
+        # The only assertion that needs a model at run time.
+        return {"type": "llm-rubric", "value": str(expectation.value)}
+    return {"type": "javascript", "value": _javascript(expectation)}
+
+
+def _javascript(expectation: Expectation) -> str:
+    value = json.dumps(expectation.value)
+    tool = json.dumps(expectation.tool)
+    path = json.dumps(expectation.path)
+
+    match expectation.type:
+        case ExpectationType.OUTPUT_CONTAINS:
+            return f"{_TEXT}.includes({value})"
+        case ExpectationType.OUTPUT_NOT_CONTAINS:
+            return f"!{_TEXT}.includes({value})"
+        case ExpectationType.OUTPUT_MATCHES:
+            return f"new RegExp({value}).test({_TEXT})"
+        case ExpectationType.TOOL_CALLED:
+            return f"{_CALLS}.some(c=>c.tool==={tool})"
+        case ExpectationType.TOOL_NOT_CALLED:
+            return f"!{_CALLS}.some(c=>c.tool==={tool})"
+        case ExpectationType.TOOL_ARGUMENT_EQUALS:
+            return f"{_CALLS}.some(c=>c.tool==={tool}&&{_argument_matches(path, value)})"
+        case ExpectationType.TOOL_ARGUMENT_NOT_EQUALS:
+            # "Not equals" means no call passed that value, not "some call
+            # passed something else" -- one correct call must not excuse a
+            # second wrong one.
+            return f"!{_CALLS}.some(c=>c.tool==={tool}&&{_argument_matches(path, value)})"
+        case ExpectationType.MAX_TOOL_CALLS:
+            scope = "true" if expectation.tool is None else f"c.tool==={tool}"
+            return f"{_CALLS}.filter(c=>{scope}).length<={int(expectation.value)}"
+        case _:  # pragma: no cover - every type is handled above
+            raise ValueError(f"cannot translate {expectation.type}")
+
+
+def _argument_matches(path: str, value: str) -> str:
+    getter = f"({_ARGUMENT_GETTER})({path},c.arguments||{{}})"
+    # Compared as JSON so structured argument values behave, and so that 1 and
+    # "1" stay different.
+    return f"JSON.stringify({getter})===JSON.stringify({value})"
+
+
+def _input_text(test: RegressionTest) -> str:
+    if test.input.text:
+        return test.input.text
+    for message in test.input.messages:
+        if message.get("role") == "user":
+            return message.get("content", "")
+    return test.input.messages[0].get("content", "") if test.input.messages else ""
+
+
+def _script_path(path: str | None, project_root: Path | None, config_dir: Path | None) -> str:
+    """Rewrite a project-relative script path for the config's own location."""
+    if path is None:  # pragma: no cover - validate_shape rejects this first
+        return ""
+    if project_root is None:
+        return path
+    resolved = (project_root / path).resolve()
+    if config_dir is None:
+        return str(resolved)
+    return os.path.relpath(resolved, config_dir.resolve())

evalkeep/failures.py

@@ -0,0 +1,95 @@
+"""Failure candidates: one per trace, carrying every contributing signal.
+
+A failure is the durable record that a trace is worth attention. It holds the
+evidence the detectors found and, once a person has looked, their decision. The
+two are kept apart on purpose: signals are derived and can be recomputed, while
+a review is a human judgement that automated detection must never overwrite.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from enum import StrEnum
+
+from evalkeep.detectors import Signal
+
+FAILURE_ID_PREFIX = "fail-"
+_FAILURE_ID_LENGTH = 12
+
+
+class FailureStatus(StrEnum):
+    """Where a failure stands with its reviewer."""
+
+    #: Detected automatically; nobody has looked yet.
+    CANDIDATE = "candidate"
+    #: A person confirmed this is a real failure.
+    CONFIRMED = "confirmed"
+    #: A person decided this is not a failure. Kept for audit, not deleted.
+    DISMISSED = "dismissed"
+
+
+class FailureOrigin(StrEnum):
+    DETECTOR = "detector"
+    MANUAL = "manual"
+
+
+def failure_id_for(trace_id: str) -> str:
+    """A stable failure ID derived from the trace it belongs to.
+
+    Detection is idempotent because this is a pure function of the trace ID:
+    re-running it addresses the same row rather than creating a second one.
+    """
+    digest = hashlib.sha256(trace_id.encode("utf-8")).hexdigest()[:_FAILURE_ID_LENGTH]
+    return f"{FAILURE_ID_PREFIX}{digest}"
+
+
+@dataclass
+class Failure:
+    """One trace's failure record."""
+
+    failure_id: str
+    trace_id: str
+    status: FailureStatus = FailureStatus.CANDIDATE
+    origin: FailureOrigin = FailureOrigin.DETECTOR
+    signals: list[Signal] = field(default_factory=list)
+    detected_at: datetime = field(default_factory=lambda: datetime.now(UTC))
+    updated_at: datetime = field(default_factory=lambda: datetime.now(UTC))
+    reviewer: str | None = None
+    reason: str | None = None
+
+    @classmethod
+    def from_signals(cls, trace_id: str, signals: list[Signal]) -> Failure:
+        return cls(failure_id=failure_id_for(trace_id), trace_id=trace_id, signals=list(signals))
+
+    @classmethod
+    def manual(cls, trace_id: str, *, reviewer: str, reason: str | None = None) -> Failure:
+        """A failure a person added by hand, with no detector evidence."""
+        return cls(
+            failure_id=failure_id_for(trace_id),
+            trace_id=trace_id,
+            status=FailureStatus.CONFIRMED,
+            origin=FailureOrigin.MANUAL,
+            reviewer=reviewer,
+            reason=reason,
+        )
+
+    @property
+    def reviewed(self) -> bool:
+        """True once a person has decided, which detection must not undo."""
+        return self.status is not FailureStatus.CANDIDATE
+
+    @property
+    def kinds(self) -> list[str]:
+        """The distinct evidence kinds behind this failure, in order."""
+        seen: dict[str, None] = {}
+        for signal in self.signals:
+            seen.setdefault(signal.kind.value, None)
+        return list(seen)
+
+    def review(self, status: FailureStatus, *, reviewer: str, reason: str | None) -> None:
+        self.status = status
+        self.reviewer = reviewer
+        self.reason = reason
+        self.updated_at = datetime.now(UTC)

evalkeep/generation.py

@@ -0,0 +1,303 @@
+"""Turning a reviewed failure into a draft regression test.
+
+Every expectation here is read off the recorded trace. Nothing is inferred about
+what the agent *should* have done, because a trace does not contain that: it
+shows an action and a verdict, not an intention. So the generator writes the
+half it can defend -- "you called refund_order with order-A and that was wrong"
+-- and records, as a warning, that a reviewer still owes the other half.
+
+That is why drafts are the only thing this module produces.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from typing import Any
+
+from evalkeep.analysis import FailureAnalysis, FailureType
+from evalkeep.failures import Failure
+from evalkeep.hashing import content_hash
+from evalkeep.regression import (
+    CaseInput,
+    Expectation,
+    ExpectationType,
+    Fixture,
+    Provenance,
+    RegressionTest,
+    ReviewStatus,
+    find_contradictions,
+    make_test_id,
+    validate_expectation,
+)
+from evalkeep.trace import NormalizedTrace, ToolCallEvent, ToolResultEvent
+
+#: Bumped when generation would produce different tests from the same input.
+GENERATOR_VERSION = 1
+
+#: How deep to flatten nested tool arguments into dotted paths.
+MAX_ARGUMENT_DEPTH = 3
+
+NO_POSITIVE_EXPECTATION = (
+    "No positive expectation: this test forbids the observed mistake but does "
+    "not say what the agent should do instead. Add one before approving."
+)
+UNDESCRIBED_FAILURE = (
+    "This failure has not been described, so the test forbids the last action "
+    "the agent took without knowing whether that was the relevant mistake. "
+    "Describe it with 'evalkeep failures label' for a sharper test."
+)
+NO_TOOL_CALLS = (
+    "The failure is about tool use but the trace records no tool calls, so no "
+    "tool expectation could be derived."
+)
+
+
+def build_test(
+    trace: NormalizedTrace,
+    failure: Failure,
+    analysis: FailureAnalysis | None,
+    *,
+    cluster_id: str | None = None,
+    cluster_label: str | None = None,
+    representative_roles: list[str] | None = None,
+) -> RegressionTest:
+    """Generate one pending draft from one analyzed failure."""
+    expectations, warnings = derive_expectations(trace, analysis)
+
+    for expectation in expectations:
+        problem = validate_expectation(expectation)
+        if problem is not None:  # pragma: no cover - generation should not emit these
+            warnings.append(f"Invalid expectation ({expectation.describe()}): {problem}")
+
+    # A contradictory draft fails on a correct agent too, so it is caught here
+    # rather than surfacing later as a phantom regression.
+    for contradiction in find_contradictions(expectations):
+        warnings.append(f"Contradictory expectations: {contradiction.describe()}")
+
+    if not any(expectation.positive for expectation in expectations):
+        warnings.append(NO_POSITIVE_EXPECTATION)
+
+    return RegressionTest(
+        test_id=make_test_id(trace.trace_id, _input_text(trace)),
+        failure_id=failure.failure_id,
+        status=ReviewStatus.DRAFT,
+        input=_build_input(trace),
+        fixtures=_build_fixtures(trace),
+        expectations=expectations,
+        warnings=warnings,
+        provenance=Provenance(
+            trace_id=trace.trace_id,
+            failure_id=failure.failure_id,
+            content_hash=content_hash(trace),
+            cluster_id=cluster_id,
+            cluster_label=cluster_label,
+            representative_roles=list(representative_roles or []),
+            failure_type=analysis.failure_type.value if analysis else None,
+            severity=analysis.severity.value if analysis else None,
+            analyzer=analysis.analyzer if analysis else None,
+            analysis_summary=analysis.summary if analysis else None,
+            evidence=[signal.kind.value for signal in failure.signals],
+            generator_version=GENERATOR_VERSION,
+        ),
+    )
+
+
+def derive_expectations(
+    trace: NormalizedTrace, analysis: FailureAnalysis | None
+) -> tuple[list[Expectation], list[str]]:
+    """Read expectations off the trace, deterministic ones first."""
+    calls = trace.tool_calls
+    warnings: list[str] = []
+    expectations: list[Expectation] = []
+
+    if analysis is None:
+        # Nobody has said what kind of failure this is, but detection found
+        # evidence that it *is* one, and the trace still shows exactly what the
+        # agent did. Forbidding that specific action needs no diagnosis -- it is
+        # the same forbidding half a described failure gets, minus the
+        # confidence that this was the relevant mistake.
+        warnings.append(UNDESCRIBED_FAILURE)
+        expectations += _forbid_observed_arguments(calls, warnings)
+        if not expectations:
+            expectations.append(
+                Expectation(
+                    type=ExpectationType.HUMAN_RUBRIC,
+                    value=_rubric_from_evidence(failure_evidence(trace)),
+                )
+            )
+        return expectations, warnings
+
+    match analysis.failure_type:
+        case FailureType.WRONG_TOOL_ARGUMENT:
+            expectations += _forbid_observed_arguments(calls, warnings)
+        case FailureType.WRONG_TOOL_SELECTION:
+            expectations += _forbid_observed_tool(calls, warnings)
+        case FailureType.UNNECESSARY_ACTION:
+            expectations += _limit_repeated_calls(calls, warnings)
+        case _:
+            warnings.append(
+                f"No deterministic expectation can be derived from a "
+                f"{analysis.failure_type.value} failure; it needs a reviewer's check."
+            )
+
+    if not expectations:
+        # The rubric is a last resort: it costs an LLM judge at run time, so it
+        # only appears where nothing checkable could be read off the trace.
+        expectations.append(
+            Expectation(
+                type=ExpectationType.HUMAN_RUBRIC,
+                value=f"The agent must not repeat this failure: {analysis.summary}",
+            )
+        )
+    return expectations, warnings
+
+
+def failure_evidence(trace: NormalizedTrace) -> str:
+    """What the trace itself says went wrong, for an undescribed failure."""
+    feedback = trace.outcome.feedback
+    if feedback is not None and feedback.comment:
+        return feedback.comment
+    for evaluation in trace.outcome.evaluations:
+        if evaluation.passed is False and evaluation.reason:
+            return evaluation.reason
+    return "the interaction was recorded as a failure"
+
+
+def _rubric_from_evidence(evidence: str) -> str:
+    return f"The agent must not repeat this failure: {evidence}"
+
+
+def _forbid_observed_arguments(
+    calls: list[ToolCallEvent], warnings: list[str]
+) -> list[Expectation]:
+    """The last call is the action the outcome is about.
+
+    A heuristic, and a documented one: earlier calls in a trace are usually
+    lookups that succeeded, and asserting against their arguments would forbid
+    correct behaviour. The reviewer can widen this during review.
+    """
+    if not calls:
+        warnings.append(NO_TOOL_CALLS)
+        return []
+
+    implicated = calls[-1]
+    arguments = {
+        path: value
+        for path, value in _flatten(implicated.arguments).items()
+        if not _is_prose(value)
+    }
+    if not arguments:
+        # Every argument was free-form prose. "summary != '<300 words>'" is
+        # satisfied by any rewording, so it is a check that cannot fail --
+        # worse than no check, because it looks like coverage. Forbid the call
+        # itself instead: on a trace like an unnecessary escalation, making it
+        # at all is the mistake, not the wording of its argument.
+        warnings.append(
+            f"{implicated.tool} was called with no argument specific enough to assert "
+            f"against, so the test forbids the call itself. Narrow it at review if the "
+            f"call was right and only its arguments were wrong."
+        )
+        return _forbid_observed_tool(calls, [])
+    if len(calls) > 1:
+        warnings.append(
+            f"Assertions target the last tool call ({implicated.tool}); "
+            f"{len(calls) - 1} earlier call(s) were treated as context."
+        )
+    return [
+        Expectation(
+            type=ExpectationType.TOOL_ARGUMENT_NOT_EQUALS,
+            tool=implicated.tool,
+            path=path,
+            value=value,
+        )
+        for path, value in arguments.items()
+    ]
+
+
+#: Longer than any identifier, code, enum or amount an argument carries, and
+#: about where a value stops being something a second run could reproduce.
+_PROSE_LENGTH = 80
+
+
+def _is_prose(value: Any) -> bool:
+    """True for a free-text argument, which no equality check can pin down."""
+    return isinstance(value, str) and (len(value) > _PROSE_LENGTH or value.count(" ") > 8)
+
+
+def _forbid_observed_tool(calls: list[ToolCallEvent], warnings: list[str]) -> list[Expectation]:
+    if not calls:
+        warnings.append(NO_TOOL_CALLS)
+        return []
+    return [Expectation(type=ExpectationType.TOOL_NOT_CALLED, tool=calls[-1].tool)]
+
+
+def _limit_repeated_calls(calls: list[ToolCallEvent], warnings: list[str]) -> list[Expectation]:
+    """Cap the tool the agent over-used at one fewer than it managed.
+
+    Deliberately a weak bound. The trace proves N calls was too many; it does
+    not prove what the right number is, and picking one would be inventing the
+    reviewer's judgement.
+    """
+    if not calls:
+        warnings.append(NO_TOOL_CALLS)
+        return []
+
+    counts = Counter(call.tool for call in calls)
+    tool, count = counts.most_common(1)[0]
+    if count == 1:
+        return [Expectation(type=ExpectationType.TOOL_NOT_CALLED, tool=tool)]
+
+    warnings.append(
+        f"{tool} was called {count} times; the limit is set to {count - 1}, "
+        "which only proves the observed count was too many. Tighten it during review."
+    )
+    return [Expectation(type=ExpectationType.MAX_TOOL_CALLS, tool=tool, value=count - 1)]
+
+
+def _build_input(trace: NormalizedTrace) -> CaseInput:
+    return CaseInput(
+        text=trace.input.text,
+        messages=[
+            {"role": message.role.value, "content": message.content}
+            for message in trace.input.messages
+        ],
+    )
+
+
+def _input_text(trace: NormalizedTrace) -> str:
+    if trace.input.text:
+        return trace.input.text
+    for message in trace.input.messages:
+        if message.role.value == "user":
+            return message.content
+    return trace.input.messages[0].content if trace.input.messages else ""
+
+
+def _build_fixtures(trace: NormalizedTrace) -> list[Fixture]:
+    """The tool results the original agent saw, paired with the calls that got them."""
+    arguments_by_call: dict[str, dict[str, Any]] = {
+        call.call_id: call.arguments for call in trace.tool_calls if call.call_id
+    }
+    return [
+        Fixture(
+            tool=event.tool,
+            arguments=arguments_by_call.get(event.call_id or "", {}),
+            result=event.result,
+            error=event.error,
+            call_id=event.call_id,
+        )
+        for event in trace.events
+        if isinstance(event, ToolResultEvent)
+    ]
+
+
+def _flatten(arguments: dict[str, Any], prefix: str = "", depth: int = 0) -> dict[str, Any]:
+    """Scalar arguments as dotted paths, so nested values are still assertable."""
+    flat: dict[str, Any] = {}
+    for key, value in arguments.items():
+        path = f"{prefix}{key}"
+        if isinstance(value, dict) and depth < MAX_ARGUMENT_DEPTH:
+            flat.update(_flatten(value, prefix=f"{path}.", depth=depth + 1))
+        elif isinstance(value, str | int | float | bool) or value is None:
+            flat[path] = value
+    return flat

evalkeep/hashing.py

@@ -0,0 +1,56 @@
+"""Stable content hashing, for detecting the same interaction twice.
+
+Two traces are the same *interaction* when the agent was asked the same thing,
+did the same things and got the same verdict -- regardless of when it was
+recorded, what the exporter called the events, or which run it came from. The
+hash therefore covers what failure analysis actually reads and deliberately
+excludes:
+
+* ``trace_id`` -- the question is whether a re-export under a new ID is the same
+  interaction, so the ID cannot be part of the answer.
+* ``metadata`` and every ``timestamp`` -- wall-clock time and provenance differ
+  between exports of one interaction.
+* ``event_id`` and ``call_id`` -- arbitrary handles, often regenerated per export.
+
+Hashing runs on the *redacted* trace, so an identical interaction hashes the same
+whether or not it happened to carry a customer's email address.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import Any
+
+from evalkeep.trace import NormalizedTrace
+
+HASH_ALGORITHM = "sha256"
+
+#: Per-event keys that identify the recording rather than the interaction.
+_VOLATILE_EVENT_KEYS = frozenset({"event_id", "call_id", "timestamp"})
+
+
+def canonical_content(trace: NormalizedTrace) -> dict[str, Any]:
+    """The subset of a trace that defines the interaction."""
+    payload = trace.model_dump(mode="json")
+    return {
+        "input": payload["input"],
+        "output": payload["output"],
+        "outcome": payload["outcome"],
+        "events": [
+            {key: value for key, value in event.items() if key not in _VOLATILE_EVENT_KEYS}
+            for event in payload["events"]
+        ],
+    }
+
+
+def content_hash(trace: NormalizedTrace) -> str:
+    """A stable ``sha256:<hex>`` fingerprint of the interaction."""
+    canonical = json.dumps(
+        canonical_content(trace),
+        sort_keys=True,
+        separators=(",", ":"),
+        ensure_ascii=False,
+    )
+    digest = hashlib.new(HASH_ALGORITHM, canonical.encode("utf-8")).hexdigest()
+    return f"{HASH_ALGORITHM}:{digest}"