agentprdiff 0.1.0__py3-none-any.whl

agentprdiff/__init__.py

@@ -0,0 +1,81 @@
+"""agentprdiff — snapshot testing for LLM agents.
+
+The one-happy-path public API:
+
+    from agentprdiff import suite, case
+    from agentprdiff.graders import contains, tool_called, latency_lt_ms, semantic
+
+    def my_agent(query: str) -> str:
+        ...
+
+    billing_suite = suite(
+        name="billing",
+        agent=my_agent,
+        cases=[
+            case(
+                name="refund_happy_path",
+                input="I want a refund for order #1234",
+                expect=[
+                    contains("refund"),
+                    tool_called("lookup_order"),
+                    semantic("agent acknowledges the refund and provides next steps"),
+                    latency_lt_ms(10_000),
+                ],
+            ),
+        ],
+    )
+
+Run from the shell::
+
+    agentprdiff init
+    agentprdiff record path/to/my_suite.py     # save baselines
+    agentprdiff check  path/to/my_suite.py     # diff against baselines; exit 1 on regression
+"""
+
+from __future__ import annotations
+
+from .core import (
+    AgentFn,
+    Case,
+    Grader,
+    GradeResult,
+    LLMCall,
+    Suite,
+    ToolCall,
+    Trace,
+    case,
+    run_agent,
+    suite,
+)
+from .differ import AssertionChange, TraceDelta, diff_traces
+from .runner import CaseReport, Runner, RunReport
+from .store import BaselineStore
+
+__version__ = "0.1.0"
+
+__all__ = [
+    # core
+    "Suite",
+    "Case",
+    "Trace",
+    "LLMCall",
+    "ToolCall",
+    "Grader",
+    "GradeResult",
+    "AgentFn",
+    "suite",
+    "case",
+    "run_agent",
+    # diffing
+    "TraceDelta",
+    "AssertionChange",
+    "diff_traces",
+    # runner
+    "Runner",
+    "RunReport",
+    "CaseReport",
+    # storage
+    "BaselineStore",
+    # version
+    "__version__",
+]

agentprdiff/cli.py

@@ -0,0 +1,124 @@
+"""Command-line interface for agentprdiff.
+
+Four subcommands:
+
+* `agentprdiff init`    — scaffold a .agentprdiff/ directory
+* `agentprdiff record`  — record baselines for every suite in a file
+* `agentprdiff check`   — compare against baselines; exit 1 on regression
+* `agentprdiff diff`    — show the diff for the most recent run of a case
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import click
+
+from .loader import load_suites
+from .reporters import JsonReporter, TerminalReporter
+from .runner import Runner
+from .store import BaselineStore
+
+
+@click.group(help="Snapshot testing for LLM agents.")
+@click.version_option(package_name="agentprdiff", prog_name="agentprdiff")
+@click.option(
+    "--root",
+    default=".agentprdiff",
+    show_default=True,
+    help="Directory where baselines and runs are stored.",
+)
+@click.pass_context
+def main(ctx: click.Context, root: str) -> None:
+    ctx.ensure_object(dict)
+    ctx.obj["store"] = BaselineStore(root=root)
+
+
+@main.command("init")
+@click.pass_context
+def cmd_init(ctx: click.Context) -> None:
+    """Create the .agentprdiff/ directory and a starter .gitignore."""
+    store: BaselineStore = ctx.obj["store"]
+    store.ensure_initialized()
+    click.echo(f"initialized {store.root}/")
+    click.echo(f"  baselines: {store.baselines_dir}/   (commit this)")
+    click.echo(f"  runs:      {store.runs_dir}/        (gitignored)")
+
+
+@main.command("record")
+@click.argument("suite_file", type=click.Path(exists=True, dir_okay=False, path_type=Path))
+@click.option("--json-out", type=click.Path(path_type=Path), help="Write JSON report to this path.")
+@click.pass_context
+def cmd_record(ctx: click.Context, suite_file: Path, json_out: Path | None) -> None:
+    """Run every suite in SUITE_FILE and save each trace as the baseline."""
+    store: BaselineStore = ctx.obj["store"]
+    runner = Runner(store)
+    terminal = TerminalReporter()
+
+    suites = load_suites(suite_file)
+    any_error = False
+    for s in suites:
+        report = runner.record(s)
+        terminal.render(report)
+        if json_out:
+            JsonReporter().render(report, json_out)
+        # record mode doesn't fail on failures, but a literal exception during
+        # execution still warrants a nonzero exit.
+        if any(cr.trace.error for cr in report.case_reports):
+            any_error = True
+
+    sys.exit(1 if any_error else 0)
+
+
+@main.command("check")
+@click.argument("suite_file", type=click.Path(exists=True, dir_okay=False, path_type=Path))
+@click.option("--json-out", type=click.Path(path_type=Path), help="Write JSON report to this path.")
+@click.option(
+    "--fail-on/--no-fail-on",
+    "fail_on_regression",
+    default=True,
+    show_default=True,
+    help="Exit non-zero when regressions are detected.",
+)
+@click.pass_context
+def cmd_check(
+    ctx: click.Context,
+    suite_file: Path,
+    json_out: Path | None,
+    fail_on_regression: bool,
+) -> None:
+    """Run every suite in SUITE_FILE and diff against saved baselines."""
+    store: BaselineStore = ctx.obj["store"]
+    runner = Runner(store)
+    terminal = TerminalReporter()
+
+    any_regression = False
+    suites = load_suites(suite_file)
+    for s in suites:
+        report = runner.check(s)
+        terminal.render(report)
+        if json_out:
+            JsonReporter().render(report, json_out)
+        any_regression = any_regression or report.has_regression
+
+    sys.exit(1 if (any_regression and fail_on_regression) else 0)
+
+
+@main.command("diff")
+@click.argument("suite_name")
+@click.argument("case_name")
+@click.pass_context
+def cmd_diff(ctx: click.Context, suite_name: str, case_name: str) -> None:
+    """Show the saved baseline trace for SUITE_NAME / CASE_NAME as pretty JSON."""
+    store: BaselineStore = ctx.obj["store"]
+    trace = store.load_baseline(suite_name, case_name)
+    if trace is None:
+        click.echo(f"no baseline found for {suite_name}/{case_name}", err=True)
+        sys.exit(2)
+    click.echo(json.dumps(trace.model_dump(mode="json"), indent=2))
+
+
+if __name__ == "__main__":  # pragma: no cover
+    main()

agentprdiff/core.py

@@ -0,0 +1,217 @@
+"""Core data model for agentprdiff.
+
+A `Suite` is a list of `Case`s. Each case, when run, produces a `Trace`. A
+`Grader` is a callable `(Trace) -> GradeResult` that asserts something about
+the trace. The test result for a case is the logical AND of every grader's
+result.
+
+Traces are serializable (JSON via pydantic) so they can be stored as baselines
+and diffed across runs.
+"""
+
+from __future__ import annotations
+
+import time
+import uuid
+from collections.abc import Callable
+from datetime import datetime, timezone
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+# ---------------------------------------------------------------------------
+# Trace model — what an agent run produces.
+# ---------------------------------------------------------------------------
+
+
+class LLMCall(BaseModel):
+    """A single model invocation captured during an agent run."""
+
+    provider: str
+    model: str
+    input_messages: list[dict[str, Any]] = Field(default_factory=list)
+    output_text: str = ""
+    tool_calls: list[dict[str, Any]] = Field(default_factory=list)
+    prompt_tokens: int = 0
+    completion_tokens: int = 0
+    cost_usd: float = 0.0
+    latency_ms: float = 0.0
+    timestamp: str = ""
+
+
+class ToolCall(BaseModel):
+    """A single tool / function invocation captured during an agent run."""
+
+    name: str
+    arguments: dict[str, Any] = Field(default_factory=dict)
+    result: Any = None
+    latency_ms: float = 0.0
+    error: str | None = None
+
+
+class Trace(BaseModel):
+    """Full record of one agent run.
+
+    A trace is what we record as a baseline and what we diff against on
+    subsequent runs. Keep it JSON-serializable.
+    """
+
+    model_config = ConfigDict(extra="allow")
+
+    case_name: str
+    suite_name: str
+    input: Any
+    output: Any = None
+    llm_calls: list[LLMCall] = Field(default_factory=list)
+    tool_calls: list[ToolCall] = Field(default_factory=list)
+    total_cost_usd: float = 0.0
+    total_latency_ms: float = 0.0
+    total_prompt_tokens: int = 0
+    total_completion_tokens: int = 0
+    error: str | None = None
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+    # A stable id (per-run, not per-case) is handy for telemetry / ci logs.
+    run_id: str = Field(default_factory=lambda: uuid.uuid4().hex[:12])
+    created_at: str = Field(
+        default_factory=lambda: datetime.now(timezone.utc).isoformat(timespec="seconds")
+    )
+
+    def record_llm_call(self, call: LLMCall) -> None:
+        self.llm_calls.append(call)
+        self.total_cost_usd += call.cost_usd
+        self.total_latency_ms += call.latency_ms
+        self.total_prompt_tokens += call.prompt_tokens
+        self.total_completion_tokens += call.completion_tokens
+
+    def record_tool_call(self, call: ToolCall) -> None:
+        self.tool_calls.append(call)
+        self.total_latency_ms += call.latency_ms
+
+
+# ---------------------------------------------------------------------------
+# Grader model — what's being asserted about a trace.
+# ---------------------------------------------------------------------------
+
+
+class GradeResult(BaseModel):
+    """The outcome of a single grader on a single trace."""
+
+    passed: bool
+    grader_name: str
+    reason: str = ""
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+# A Grader is any callable (Trace) -> GradeResult. We keep it as a simple
+# callable rather than an abstract class so users can pass lambdas.
+Grader = Callable[[Trace], GradeResult]
+
+
+# ---------------------------------------------------------------------------
+# Case and Suite.
+# ---------------------------------------------------------------------------
+
+
+class Case(BaseModel):
+    """One input + the assertions that must hold for the resulting trace."""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    name: str
+    input: Any
+    expect: list[Grader] = Field(default_factory=list)
+    tags: list[str] = Field(default_factory=list)
+
+
+# An Agent is any callable `(input) -> (output, Trace)`. If the user's agent
+# returns only an output, the runner wraps it so latency is captured but the
+# returned `Trace` has empty llm_calls / tool_calls.
+AgentFn = Callable[[Any], Any]
+
+
+class Suite(BaseModel):
+    """A named group of cases sharing one agent under test."""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    name: str
+    agent: AgentFn
+    cases: list[Case]
+    description: str = ""
+
+
+# ---------------------------------------------------------------------------
+# Public constructors — keep them short and opinionated.
+# ---------------------------------------------------------------------------
+
+
+def suite(name: str, agent: AgentFn, cases: list[Case], description: str = "") -> Suite:
+    """Create a Suite.
+
+    >>> s = suite("billing", my_agent, cases=[case(...)])
+    """
+    return Suite(name=name, agent=agent, cases=cases, description=description)
+
+
+def case(name: str, input: Any, expect: list[Grader], tags: list[str] | None = None) -> Case:
+    """Create a Case.
+
+    >>> c = case("refund", input="I want a refund", expect=[contains("refund")])
+    """
+    return Case(name=name, input=input, expect=expect or [], tags=tags or [])
+
+
+# ---------------------------------------------------------------------------
+# Small helper: run an agent callable and build a Trace.
+# ---------------------------------------------------------------------------
+
+
+def run_agent(
+    agent: AgentFn,
+    *,
+    suite_name: str,
+    case_name: str,
+    input_value: Any,
+) -> Trace:
+    """Invoke an agent callable and build a Trace around its execution.
+
+    If the agent returns a `(output, Trace)` tuple, we use the returned trace
+    and just fill in the metadata we can see from out here (suite/case names).
+    Otherwise we build a minimal trace with latency only.
+    """
+    start = time.perf_counter()
+    trace: Trace
+    try:
+        result = agent(input_value)
+    except Exception as exc:  # noqa: BLE001 — we want to capture any failure mode
+        elapsed_ms = (time.perf_counter() - start) * 1000.0
+        return Trace(
+            suite_name=suite_name,
+            case_name=case_name,
+            input=input_value,
+            output=None,
+            error=f"{type(exc).__name__}: {exc}",
+            total_latency_ms=elapsed_ms,
+        )
+    elapsed_ms = (time.perf_counter() - start) * 1000.0
+
+    if isinstance(result, tuple) and len(result) == 2 and isinstance(result[1], Trace):
+        output, trace = result
+        trace.suite_name = suite_name
+        trace.case_name = case_name
+        trace.input = input_value
+        trace.output = output
+        # If the agent didn't track latency itself, fill in wall time as a
+        # floor.
+        if trace.total_latency_ms == 0.0:
+            trace.total_latency_ms = elapsed_ms
+    else:
+        trace = Trace(
+            suite_name=suite_name,
+            case_name=case_name,
+            input=input_value,
+            output=result,
+            total_latency_ms=elapsed_ms,
+        )
+    return trace

agentprdiff/differ.py

@@ -0,0 +1,161 @@
+"""Trace diffing.
+
+Given a baseline `Trace` and a current `Trace`, compute a `TraceDelta` that
+summarizes what changed. The delta is the unit of CI output:
+
+* assertion changes (which graders flipped pass->fail or fail->pass)
+* cost / latency / token deltas
+* tool-call sequence changes
+* output change (textual diff)
+
+The `regressions` property is the canonical "should CI fail?" signal.
+"""
+
+from __future__ import annotations
+
+import difflib
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from .core import GradeResult, Trace
+
+
+class AssertionChange(BaseModel):
+    grader_name: str
+    baseline_passed: bool | None  # None = grader didn't exist in baseline
+    current_passed: bool
+    current_reason: str = ""
+
+    @property
+    def is_regression(self) -> bool:
+        """A regression = was passing (or absent), now failing."""
+        return (self.baseline_passed is None or self.baseline_passed) and not self.current_passed
+
+    @property
+    def is_improvement(self) -> bool:
+        return bool(self.baseline_passed is False and self.current_passed)
+
+
+class TraceDelta(BaseModel):
+    """Summary of the delta between baseline and current trace for one case."""
+
+    suite_name: str
+    case_name: str
+
+    baseline_exists: bool
+    assertion_changes: list[AssertionChange] = Field(default_factory=list)
+
+    cost_delta_usd: float = 0.0
+    latency_delta_ms: float = 0.0
+    prompt_tokens_delta: int = 0
+    completion_tokens_delta: int = 0
+
+    tool_sequence_changed: bool = False
+    baseline_tool_sequence: list[str] = Field(default_factory=list)
+    current_tool_sequence: list[str] = Field(default_factory=list)
+
+    output_changed: bool = False
+    output_diff: str = ""
+
+    current_error: str | None = None
+    baseline_error: str | None = None
+
+    @property
+    def regressions(self) -> list[AssertionChange]:
+        return [c for c in self.assertion_changes if c.is_regression]
+
+    @property
+    def improvements(self) -> list[AssertionChange]:
+        return [c for c in self.assertion_changes if c.is_improvement]
+
+    @property
+    def has_regression(self) -> bool:
+        return bool(self.regressions) or (self.current_error is not None and self.baseline_error is None)
+
+
+def diff_traces(
+    *,
+    baseline: Trace | None,
+    current: Trace,
+    current_results: list[GradeResult],
+    baseline_results: list[GradeResult] | None = None,
+) -> TraceDelta:
+    """Build a TraceDelta.
+
+    `current_results` are the grader results from running the cases's expects
+    against the `current` trace.
+
+    `baseline_results` are optional. If provided, they're used to determine
+    per-assertion regressions directly. If omitted, we replay the same grader
+    names from `current_results` against the baseline by looking them up by
+    `grader_name` in the baseline metadata (not typically populated, so in
+    practice you should pass them).
+    """
+    delta = TraceDelta(
+        suite_name=current.suite_name,
+        case_name=current.case_name,
+        baseline_exists=baseline is not None,
+        current_error=current.error,
+        baseline_error=baseline.error if baseline else None,
+    )
+
+    baseline_by_name: dict[str, bool] = {}
+    if baseline_results:
+        for r in baseline_results:
+            baseline_by_name[r.grader_name] = r.passed
+
+    for r in current_results:
+        delta.assertion_changes.append(
+            AssertionChange(
+                grader_name=r.grader_name,
+                baseline_passed=baseline_by_name.get(r.grader_name),
+                current_passed=r.passed,
+                current_reason=r.reason,
+            )
+        )
+
+    if baseline is not None:
+        delta.cost_delta_usd = current.total_cost_usd - baseline.total_cost_usd
+        delta.latency_delta_ms = current.total_latency_ms - baseline.total_latency_ms
+        delta.prompt_tokens_delta = (
+            current.total_prompt_tokens - baseline.total_prompt_tokens
+        )
+        delta.completion_tokens_delta = (
+            current.total_completion_tokens - baseline.total_completion_tokens
+        )
+        delta.baseline_tool_sequence = [c.name for c in baseline.tool_calls]
+        delta.current_tool_sequence = [c.name for c in current.tool_calls]
+        delta.tool_sequence_changed = (
+            delta.baseline_tool_sequence != delta.current_tool_sequence
+        )
+        baseline_out = _to_str(baseline.output)
+        current_out = _to_str(current.output)
+        if baseline_out != current_out:
+            delta.output_changed = True
+            delta.output_diff = _unified_diff(baseline_out, current_out)
+
+    return delta
+
+
+def _to_str(value: Any) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value
+    try:
+        return str(value)
+    except Exception:  # noqa: BLE001
+        return ""
+
+
+def _unified_diff(a: str, b: str, n: int = 3) -> str:
+    return "".join(
+        difflib.unified_diff(
+            a.splitlines(keepends=True),
+            b.splitlines(keepends=True),
+            fromfile="baseline",
+            tofile="current",
+            n=n,
+        )
+    )

agentprdiff/graders/__init__.py

@@ -0,0 +1,38 @@
+"""Graders — assertions over a `Trace`.
+
+Every grader returns a `GradeResult`. Deterministic graders (cheap, free,
+no LLM calls) live in `deterministic.py`. The `semantic` grader uses an
+LLM-as-judge backend — it lives in `semantic.py` and is pluggable.
+
+The names exposed from this module form `agentprdiff`'s public assertion API.
+Keep the surface area small and stable.
+"""
+
+from __future__ import annotations
+
+from .deterministic import (
+    contains,
+    contains_any,
+    cost_lt_usd,
+    latency_lt_ms,
+    no_tool_called,
+    output_length_lt,
+    regex_match,
+    tool_called,
+    tool_sequence,
+)
+from .semantic import fake_judge, semantic
+
+__all__ = [
+    "contains",
+    "contains_any",
+    "regex_match",
+    "tool_called",
+    "tool_sequence",
+    "no_tool_called",
+    "output_length_lt",
+    "latency_lt_ms",
+    "cost_lt_usd",
+    "semantic",
+    "fake_judge",
+]