agentprdiff 0.1.0__py3-none-any.whl

agentprdiff/graders/deterministic.py

@@ -0,0 +1,186 @@
+"""Deterministic graders — cheap, free, reproducible.
+
+These never call an LLM. Prefer them whenever the assertion can be expressed
+mechanically; reserve the semantic grader for things you genuinely can't
+encode as a rule.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Sequence
+
+from ..core import Grader, GradeResult, Trace
+
+
+def _output_str(trace: Trace) -> str:
+    """Best-effort stringification of the agent's final output."""
+    out = trace.output
+    if out is None:
+        return ""
+    if isinstance(out, str):
+        return out
+    try:
+        return str(out)
+    except Exception:  # noqa: BLE001
+        return ""
+
+
+def contains(substring: str, *, case_sensitive: bool = False) -> Grader:
+    """Pass iff the agent's final output contains `substring`."""
+
+    def _grader(trace: Trace) -> GradeResult:
+        haystack = _output_str(trace)
+        passed = (
+            substring in haystack if case_sensitive else substring.lower() in haystack.lower()
+        )
+        return GradeResult(
+            passed=passed,
+            grader_name=f"contains({substring!r})",
+            reason=(
+                f"output {'contains' if passed else 'does not contain'} {substring!r}"
+            ),
+        )
+
+    return _grader
+
+
+def contains_any(substrings: Sequence[str], *, case_sensitive: bool = False) -> Grader:
+    """Pass iff the output contains at least one of the listed substrings."""
+
+    def _grader(trace: Trace) -> GradeResult:
+        haystack = _output_str(trace) if case_sensitive else _output_str(trace).lower()
+        needles = list(substrings) if case_sensitive else [s.lower() for s in substrings]
+        matched = [n for n in needles if n in haystack]
+        passed = bool(matched)
+        return GradeResult(
+            passed=passed,
+            grader_name=f"contains_any({list(substrings)!r})",
+            reason=(
+                f"matched {matched!r}"
+                if passed
+                else f"none of {list(substrings)!r} found in output"
+            ),
+        )
+
+    return _grader
+
+
+def regex_match(pattern: str, *, flags: int = 0) -> Grader:
+    """Pass iff `pattern` matches the agent's final output."""
+    compiled = re.compile(pattern, flags=flags)
+
+    def _grader(trace: Trace) -> GradeResult:
+        haystack = _output_str(trace)
+        m = compiled.search(haystack)
+        passed = m is not None
+        return GradeResult(
+            passed=passed,
+            grader_name=f"regex_match({pattern!r})",
+            reason=(
+                f"matched {m.group(0)!r}" if m else f"no match for {pattern!r}"
+            ),
+        )
+
+    return _grader
+
+
+def tool_called(name: str, *, min_times: int = 1) -> Grader:
+    """Pass iff the tool `name` was called at least `min_times` times."""
+
+    def _grader(trace: Trace) -> GradeResult:
+        count = sum(1 for c in trace.tool_calls if c.name == name)
+        passed = count >= min_times
+        return GradeResult(
+            passed=passed,
+            grader_name=f"tool_called({name!r}, min_times={min_times})",
+            reason=f"tool {name!r} called {count} time(s), required >= {min_times}",
+        )
+
+    return _grader
+
+
+def no_tool_called(name: str) -> Grader:
+    """Pass iff the tool `name` was NOT called."""
+
+    def _grader(trace: Trace) -> GradeResult:
+        count = sum(1 for c in trace.tool_calls if c.name == name)
+        passed = count == 0
+        return GradeResult(
+            passed=passed,
+            grader_name=f"no_tool_called({name!r})",
+            reason=f"tool {name!r} called {count} time(s); expected 0",
+        )
+
+    return _grader
+
+
+def tool_sequence(sequence: Sequence[str], *, strict: bool = False) -> Grader:
+    """Pass iff the tool-call sequence matches `sequence`.
+
+    If `strict=False` (default), `sequence` must appear as a subsequence of
+    the actual tool calls (other tools may be interleaved). If `strict=True`,
+    the tool calls must equal `sequence` exactly.
+    """
+
+    def _grader(trace: Trace) -> GradeResult:
+        actual = [c.name for c in trace.tool_calls]
+        if strict:
+            passed = actual == list(sequence)
+        else:
+            # subsequence check
+            i = 0
+            for call in actual:
+                if i < len(sequence) and call == sequence[i]:
+                    i += 1
+            passed = i == len(sequence)
+        return GradeResult(
+            passed=passed,
+            grader_name=f"tool_sequence({list(sequence)!r}, strict={strict})",
+            reason=f"actual tool sequence: {actual}",
+        )
+
+    return _grader
+
+
+def output_length_lt(max_chars: int) -> Grader:
+    """Pass iff the output has fewer than `max_chars` characters."""
+
+    def _grader(trace: Trace) -> GradeResult:
+        n = len(_output_str(trace))
+        passed = n < max_chars
+        return GradeResult(
+            passed=passed,
+            grader_name=f"output_length_lt({max_chars})",
+            reason=f"output length {n} chars, limit {max_chars}",
+        )
+
+    return _grader
+
+
+def latency_lt_ms(max_ms: float) -> Grader:
+    """Pass iff the trace's total latency is below `max_ms` milliseconds."""
+
+    def _grader(trace: Trace) -> GradeResult:
+        passed = trace.total_latency_ms < max_ms
+        return GradeResult(
+            passed=passed,
+            grader_name=f"latency_lt_ms({max_ms})",
+            reason=f"latency {trace.total_latency_ms:.1f} ms, limit {max_ms:.1f} ms",
+        )
+
+    return _grader
+
+
+def cost_lt_usd(max_usd: float) -> Grader:
+    """Pass iff the trace's total cost is below `max_usd` dollars."""
+
+    def _grader(trace: Trace) -> GradeResult:
+        passed = trace.total_cost_usd < max_usd
+        return GradeResult(
+            passed=passed,
+            grader_name=f"cost_lt_usd({max_usd})",
+            reason=f"cost ${trace.total_cost_usd:.4f}, limit ${max_usd:.4f}",
+        )
+
+    return _grader

agentprdiff/graders/semantic.py

@@ -0,0 +1,180 @@
+"""Semantic grader — LLM-as-judge.
+
+The `semantic` grader accepts a rubric (natural language) and a `judge`
+callable that returns a pass/fail verdict with a reason. We ship two built-in
+judges:
+
+* `fake_judge` — deterministic; used in tests and CI when you want a green
+  pipeline without API keys. It passes iff any rubric keyword appears in the
+  agent's output.
+* `openai_judge(model=...)` and `anthropic_judge(model=...)` — thin wrappers
+  over the respective SDKs. Imported lazily so agentprdiff has no required
+  runtime dependency on either SDK.
+
+Custom judges are encouraged. A judge is a callable:
+
+    judge(rubric: str, trace: Trace) -> (passed: bool, reason: str)
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Callable
+
+from ..core import Grader, GradeResult, Trace
+
+Judge = Callable[[str, Trace], tuple[bool, str]]
+
+
+# ---------------------------------------------------------------------------
+# Public grader.
+# ---------------------------------------------------------------------------
+
+
+def semantic(rubric: str, *, judge: Judge | None = None) -> Grader:
+    """Pass iff the `judge` says the trace satisfies the `rubric`.
+
+    The rubric is natural language, e.g. "the agent acknowledged the refund
+    and provided a ticket number".
+    """
+    backend = judge or _default_judge()
+
+    def _grader(trace: Trace) -> GradeResult:
+        try:
+            passed, reason = backend(rubric, trace)
+        except Exception as exc:  # noqa: BLE001
+            return GradeResult(
+                passed=False,
+                grader_name=f"semantic({rubric!r})",
+                reason=f"judge raised {type(exc).__name__}: {exc}",
+            )
+        return GradeResult(
+            passed=passed,
+            grader_name=f"semantic({rubric!r})",
+            reason=reason,
+        )
+
+    return _grader
+
+
+# ---------------------------------------------------------------------------
+# Built-in judges.
+# ---------------------------------------------------------------------------
+
+
+def fake_judge(rubric: str, trace: Trace) -> tuple[bool, str]:
+    """Keyword-match judge for tests. Passes iff ANY rubric word (>= 4 chars)
+    appears in the agent output, case-insensitive.
+
+    Deterministic, free, and good enough for demos and CI smoke tests. Do
+    not use in production eval pipelines.
+    """
+    output = str(trace.output or "").lower()
+    keywords = [w for w in _tokenize(rubric) if len(w) >= 4]
+    matched = [w for w in keywords if w in output]
+    passed = bool(matched)
+    reason = (
+        f"fake_judge matched keywords {matched}" if passed else "fake_judge matched no keywords"
+    )
+    return passed, reason
+
+
+def _tokenize(text: str) -> list[str]:
+    import re
+
+    return re.findall(r"[a-zA-Z]+", text.lower())
+
+
+_JUDGE_PROMPT = (
+    "You are an evaluator. Given a RUBRIC and an agent's OUTPUT, decide "
+    "whether the output satisfies the rubric.\n\n"
+    "Respond with the single word PASS or FAIL on the first line, and a "
+    "one-sentence reason on the second line. Be strict: if the rubric is "
+    "not clearly satisfied, answer FAIL.\n\n"
+    "RUBRIC:\n{rubric}\n\n"
+    "OUTPUT:\n{output}\n"
+)
+
+
+def openai_judge(model: str = "gpt-4o-mini", api_key: str | None = None) -> Judge:
+    """Return a judge backed by the OpenAI Chat Completions API.
+
+    Requires the `openai` package (install with `pip install agentprdiff[openai]`).
+    """
+
+    def _judge(rubric: str, trace: Trace) -> tuple[bool, str]:
+        from openai import OpenAI  # lazy import
+
+        client = OpenAI(api_key=api_key or os.environ.get("OPENAI_API_KEY"))
+        prompt = _JUDGE_PROMPT.format(rubric=rubric, output=str(trace.output or ""))
+        resp = client.chat.completions.create(
+            model=model,
+            messages=[{"role": "user", "content": prompt}],
+            temperature=0,
+        )
+        text = resp.choices[0].message.content or ""
+        return _parse_verdict(text)
+
+    return _judge
+
+
+def anthropic_judge(model: str = "claude-haiku-4-5-20251001", api_key: str | None = None) -> Judge:
+    """Return a judge backed by the Anthropic Messages API.
+
+    Requires the `anthropic` package (install with `pip install agentprdiff[anthropic]`).
+    """
+
+    def _judge(rubric: str, trace: Trace) -> tuple[bool, str]:
+        import anthropic  # lazy import
+
+        client = anthropic.Anthropic(api_key=api_key or os.environ.get("ANTHROPIC_API_KEY"))
+        prompt = _JUDGE_PROMPT.format(rubric=rubric, output=str(trace.output or ""))
+        resp = client.messages.create(
+            model=model,
+            max_tokens=120,
+            messages=[{"role": "user", "content": prompt}],
+        )
+        text = "".join(
+            block.text for block in resp.content if getattr(block, "type", None) == "text"
+        )
+        return _parse_verdict(text)
+
+    return _judge
+
+
+def _parse_verdict(text: str) -> tuple[bool, str]:
+    lines = [ln.strip() for ln in (text or "").strip().splitlines() if ln.strip()]
+    if not lines:
+        return False, "judge returned empty response"
+    verdict = lines[0].upper()
+    reason = lines[1] if len(lines) > 1 else ""
+    if verdict.startswith("PASS"):
+        return True, reason or "judge said PASS"
+    if verdict.startswith("FAIL"):
+        return False, reason or "judge said FAIL"
+    # Be strict: anything other than an explicit PASS is a FAIL.
+    return False, f"judge returned unparseable verdict: {lines[0]!r}"
+
+
+# ---------------------------------------------------------------------------
+# Backend selection.
+# ---------------------------------------------------------------------------
+
+
+def _default_judge() -> Judge:
+    """Pick a default judge based on environment.
+
+    Order of preference:
+    * AGENTGUARD_JUDGE=fake -> fake_judge
+    * AGENTGUARD_JUDGE=openai or OPENAI_API_KEY set -> openai_judge()
+    * AGENTGUARD_JUDGE=anthropic or ANTHROPIC_API_KEY set -> anthropic_judge()
+    * otherwise -> fake_judge (so pipelines stay green in CI without keys)
+    """
+    choice = (os.environ.get("AGENTGUARD_JUDGE") or "").lower()
+    if choice == "fake":
+        return fake_judge
+    if choice == "openai" or (not choice and os.environ.get("OPENAI_API_KEY")):
+        return openai_judge()
+    if choice == "anthropic" or (not choice and os.environ.get("ANTHROPIC_API_KEY")):
+        return anthropic_judge()
+    return fake_judge

agentprdiff/loader.py

@@ -0,0 +1,47 @@
+"""Load `Suite` objects from user-provided Python files.
+
+We deliberately keep this ultra-simple for v0.1: the user points at a python
+file path (or module) and we import it; every module-level `Suite` instance
+is a suite to run.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import importlib.util
+import sys
+from pathlib import Path
+
+from .core import Suite
+
+
+def load_suites(path: str | Path) -> list[Suite]:
+    """Import `path` and return every module-level `Suite` it defines."""
+    p = Path(path).resolve()
+    if not p.exists():
+        raise FileNotFoundError(f"no such file: {p}")
+    if p.is_dir():
+        raise IsADirectoryError(
+            f"{p} is a directory; point at a .py file that defines Suites."
+        )
+
+    module_name = f"_agentprdiff_suite_{abs(hash(str(p)))}"
+    spec = importlib.util.spec_from_file_location(module_name, p)
+    if spec is None or spec.loader is None:  # pragma: no cover
+        raise ImportError(f"could not load suite file: {p}")
+    module = importlib.util.module_from_spec(spec)
+    # Ensure the file's own directory is importable (for relative helpers).
+    sys.path.insert(0, str(p.parent))
+    try:
+        spec.loader.exec_module(module)
+    finally:
+        with contextlib.suppress(ValueError):
+            sys.path.remove(str(p.parent))
+
+    suites = [v for v in vars(module).values() if isinstance(v, Suite)]
+    if not suites:
+        raise ValueError(
+            f"{p} defines no module-level Suite objects. "
+            "Use `from agentprdiff import suite` and bind the result to a variable."
+        )
+    return suites

agentprdiff/reporters.py

@@ -0,0 +1,127 @@
+"""Reporters — render RunReports for humans and for CI.
+
+`TerminalReporter` uses rich for pretty output. `JsonReporter` writes a
+stable JSON envelope you can archive as a CI artifact.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from rich.console import Console
+from rich.panel import Panel
+from rich.table import Table
+from rich.text import Text
+
+from .runner import RunReport
+
+
+class TerminalReporter:
+    def __init__(self, console: Console | None = None) -> None:
+        self.console = console or Console()
+
+    def render(self, report: RunReport) -> None:
+        header = Text()
+        header.append(f"agentprdiff {report.mode} ", style="bold cyan")
+        header.append("— suite ", style="dim")
+        header.append(f"{report.suite_name}", style="bold")
+        header.append(
+            f"  ({report.cases_passed}/{report.cases_total} passed, "
+            f"{report.cases_regressed} regressed)",
+            style="dim",
+        )
+        self.console.print(header)
+
+        table = Table(show_header=True, header_style="bold", show_lines=False, expand=True)
+        table.add_column("Case", style="bold")
+        table.add_column("Result")
+        table.add_column("Cost Δ", justify="right")
+        table.add_column("Latency Δ", justify="right")
+        table.add_column("Notes")
+
+        for cr in report.case_reports:
+            if cr.has_regression:
+                result = Text("REGRESSION", style="bold red")
+            elif cr.passed:
+                result = Text("PASS", style="bold green")
+            else:
+                result = Text("FAIL", style="bold red")
+
+            cost_cell = ""
+            latency_cell = ""
+            notes = []
+            if cr.delta is not None:
+                if cr.delta.cost_delta_usd:
+                    cost_cell = _format_delta(cr.delta.cost_delta_usd, "${:+.4f}")
+                if cr.delta.latency_delta_ms:
+                    latency_cell = _format_delta(cr.delta.latency_delta_ms, "{:+.0f} ms")
+                if cr.delta.tool_sequence_changed:
+                    notes.append(
+                        "tools: "
+                        f"{cr.delta.baseline_tool_sequence} → "
+                        f"{cr.delta.current_tool_sequence}"
+                    )
+                if cr.delta.output_changed and not cr.has_regression:
+                    notes.append("output changed")
+                for ac in cr.delta.regressions:
+                    notes.append(f"[red]{ac.grader_name}[/red] {ac.current_reason}")
+            if cr.trace.error:
+                notes.append(f"[red]error:[/red] {cr.trace.error}")
+            for r in cr.grader_results:
+                if not r.passed and cr.delta is None:
+                    notes.append(f"[red]{r.grader_name}[/red] {r.reason}")
+
+            table.add_row(cr.case_name, result, cost_cell, latency_cell, "\n".join(notes) or "—")
+
+        self.console.print(table)
+
+        # Per-regression expanded section.
+        for cr in report.case_reports:
+            if cr.has_regression and cr.delta is not None and cr.delta.output_diff:
+                self.console.print(
+                    Panel(
+                        cr.delta.output_diff,
+                        title=f"{cr.case_name}: output diff",
+                        border_style="red",
+                    )
+                )
+
+        if report.mode == "check":
+            if report.has_regression:
+                self.console.print(
+                    Text(
+                        f"\n✗ {report.cases_regressed} regression(s) detected.",
+                        style="bold red",
+                    )
+                )
+            else:
+                self.console.print(Text("\n✓ no regressions.", style="bold green"))
+
+
+class JsonReporter:
+    """Write a stable JSON envelope suitable for CI artifact archiving."""
+
+    def render(self, report: RunReport, path: Path) -> Path:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        payload = {
+            "suite": report.suite_name,
+            "mode": report.mode,
+            "summary": {
+                "cases_total": report.cases_total,
+                "cases_passed": report.cases_passed,
+                "cases_regressed": report.cases_regressed,
+                "has_regression": report.has_regression,
+            },
+            "cases": [cr.model_dump(mode="json") for cr in report.case_reports],
+        }
+        path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
+        return path
+
+
+def _format_delta(value: float, fmt: str) -> str:
+    if value == 0:
+        return ""
+    text = fmt.format(value)
+    color = "green" if value < 0 else "red"
+    return f"[{color}]{text}[/{color}]"

agentprdiff/runner.py

@@ -0,0 +1,130 @@
+"""Run a Suite — the heart of agentprdiff.
+
+Two modes:
+
+* `record` — run each case, save the resulting `Trace` as the baseline, do
+  not compare.
+* `check` — run each case, load the baseline (if any), compute a `TraceDelta`,
+  aggregate into a `RunReport`. Exit status at the CLI is driven by
+  `RunReport.has_regression`.
+"""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from .core import GradeResult, Suite, Trace, run_agent
+from .differ import TraceDelta, diff_traces
+from .store import BaselineStore
+
+
+class CaseReport(BaseModel):
+    """Per-case outcome within a RunReport."""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    suite_name: str
+    case_name: str
+    trace: Trace
+    grader_results: list[GradeResult]
+    delta: TraceDelta | None = None
+
+    @property
+    def passed(self) -> bool:
+        """All graders passed for the current run."""
+        return all(r.passed for r in self.grader_results) and self.trace.error is None
+
+    @property
+    def has_regression(self) -> bool:
+        """Whether this case regressed vs baseline. If there is no baseline,
+        we treat a full pass as not-a-regression; any failing grader is
+        treated as a regression (first-run-bad is still bad)."""
+        if not self.passed:
+            return True
+        if self.delta is not None:
+            return self.delta.has_regression
+        return False
+
+
+class RunReport(BaseModel):
+    """Aggregate result of running a suite."""
+
+    suite_name: str
+    mode: str  # "record" or "check"
+    case_reports: list[CaseReport] = Field(default_factory=list)
+
+    @property
+    def cases_passed(self) -> int:
+        return sum(1 for c in self.case_reports if c.passed)
+
+    @property
+    def cases_total(self) -> int:
+        return len(self.case_reports)
+
+    @property
+    def cases_regressed(self) -> int:
+        return sum(1 for c in self.case_reports if c.has_regression)
+
+    @property
+    def has_regression(self) -> bool:
+        return any(c.has_regression for c in self.case_reports)
+
+
+class Runner:
+    """Runs suites in record or check mode."""
+
+    def __init__(self, store: BaselineStore) -> None:
+        self.store = store
+
+    # ------------------------------------------------------------------ api
+
+    def record(self, suite: Suite) -> RunReport:
+        return self._run(suite, mode="record")
+
+    def check(self, suite: Suite) -> RunReport:
+        return self._run(suite, mode="check")
+
+    # --------------------------------------------------------------- impl
+
+    def _run(self, suite: Suite, *, mode: str) -> RunReport:
+        self.store.ensure_initialized()
+        run_id = self.store.fresh_run_id()
+        report = RunReport(suite_name=suite.name, mode=mode)
+
+        for case in suite.cases:
+            trace = run_agent(
+                suite.agent,
+                suite_name=suite.name,
+                case_name=case.name,
+                input_value=case.input,
+            )
+            grader_results = [g(trace) for g in case.expect]
+            # Persist the current run either way (record = baseline, check = runs/).
+            delta: TraceDelta | None = None
+            if mode == "record":
+                self.store.save_baseline(trace)
+            else:
+                self.store.save_run_trace(run_id, trace)
+                baseline = self.store.load_baseline(suite.name, case.name)
+                baseline_results = None
+                if baseline is not None:
+                    # Re-run graders against the baseline so the delta's
+                    # per-assertion regression flags are accurate.
+                    baseline_results = [g(baseline) for g in case.expect]
+                delta = diff_traces(
+                    baseline=baseline,
+                    current=trace,
+                    current_results=grader_results,
+                    baseline_results=baseline_results,
+                )
+
+            report.case_reports.append(
+                CaseReport(
+                    suite_name=suite.name,
+                    case_name=case.name,
+                    trace=trace,
+                    grader_results=grader_results,
+                    delta=delta,
+                )
+            )
+        return report