agentprdiff 0.1.0__tar.gz → 0.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (24) hide show
  1. {agentprdiff-0.1.0 → agentprdiff-0.2.1}/.gitignore +5 -4
  2. agentprdiff-0.2.1/CHANGELOG.md +129 -0
  3. {agentprdiff-0.1.0 → agentprdiff-0.2.1}/PKG-INFO +39 -7
  4. {agentprdiff-0.1.0 → agentprdiff-0.2.1}/README.md +35 -5
  5. agentprdiff-0.2.1/examples/regression-tour/README.md +140 -0
  6. {agentprdiff-0.1.0 → agentprdiff-0.2.1}/pyproject.toml +6 -2
  7. {agentprdiff-0.1.0 → agentprdiff-0.2.1}/src/agentprdiff/__init__.py +17 -1
  8. agentprdiff-0.2.1/src/agentprdiff/adapters/__init__.py +43 -0
  9. agentprdiff-0.2.1/src/agentprdiff/adapters/anthropic.py +191 -0
  10. agentprdiff-0.2.1/src/agentprdiff/adapters/openai.py +343 -0
  11. agentprdiff-0.2.1/src/agentprdiff/adapters/pricing.py +136 -0
  12. {agentprdiff-0.1.0 → agentprdiff-0.2.1}/src/agentprdiff/loader.py +29 -4
  13. agentprdiff-0.1.0/CHANGELOG.md +0 -37
  14. {agentprdiff-0.1.0 → agentprdiff-0.2.1}/LICENSE +0 -0
  15. {agentprdiff-0.1.0 → agentprdiff-0.2.1}/examples/quickstart/README.md +0 -0
  16. {agentprdiff-0.1.0 → agentprdiff-0.2.1}/src/agentprdiff/cli.py +0 -0
  17. {agentprdiff-0.1.0 → agentprdiff-0.2.1}/src/agentprdiff/core.py +0 -0
  18. {agentprdiff-0.1.0 → agentprdiff-0.2.1}/src/agentprdiff/differ.py +0 -0
  19. {agentprdiff-0.1.0 → agentprdiff-0.2.1}/src/agentprdiff/graders/__init__.py +0 -0
  20. {agentprdiff-0.1.0 → agentprdiff-0.2.1}/src/agentprdiff/graders/deterministic.py +0 -0
  21. {agentprdiff-0.1.0 → agentprdiff-0.2.1}/src/agentprdiff/graders/semantic.py +0 -0
  22. {agentprdiff-0.1.0 → agentprdiff-0.2.1}/src/agentprdiff/reporters.py +0 -0
  23. {agentprdiff-0.1.0 → agentprdiff-0.2.1}/src/agentprdiff/runner.py +0 -0
  24. {agentprdiff-0.1.0 → agentprdiff-0.2.1}/src/agentprdiff/store.py +0 -0
@@ -26,7 +26,8 @@ env/
26
26
  .vscode/
27
27
  *.swp
28
28
 
29
- # agentguard runtime
30
- # Users SHOULD commit .agentguard/baselines; they should NOT commit run artifacts.
31
- .agentguard/runs/
32
- .agentguard/cache/
29
+ # agentprdiff runtime
30
+ # Users SHOULD commit .agentprdiff/baselines; they should NOT commit run artifacts.
31
+ .agentprdiff/runs/
32
+ .agentprdiff/cache/
33
+ .pypirc
@@ -0,0 +1,129 @@
1
+ # Changelog
2
+
3
+ All notable changes to `agentprdiff` are documented in this file. Originally
4
+ prototyped under the name `tracediff`; renamed before first public release.
5
+
6
+ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
7
+ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
8
+
9
+ ## [0.2.1] — 2026-04-26
10
+
11
+ ### Changed
12
+
13
+ - README links to `AGENTS.md`, `docs/adapters.md`, `docs/ai-driven-adoption.md`,
14
+ `docs/suite-layout.md`, `docs/ci-integration.md`, `LICENSE`, and `CHANGELOG.md`
15
+ are now absolute GitHub URLs instead of relative paths. PyPI's project page
16
+ and Libraries.io render the README but cannot resolve relative repo paths;
17
+ the previous links rendered as broken from those surfaces. Absolute URLs
18
+ fix the click-through from PyPI / Libraries.io directly to the docs on
19
+ GitHub.
20
+ - Status section refreshed: 0.2.x is the current alpha line; OpenAI and
21
+ Anthropic SDK adapters are now shipped (previously listed as 0.2 roadmap).
22
+ LangChain/LangGraph adapters and the JS companion package moved to the
23
+ 0.3 roadmap.
24
+
25
+ ### Fixed
26
+
27
+ - Wheel metadata now emits a separate `Author:` header in addition to
28
+ `Author-email:`. Previously, the PEP 621 `authors = [{ name = ..., email = ... }]`
29
+ form generated only `Author-email: "Name" <email>`, which downstream
30
+ parsers like pypistats.org could not read (showing "Author: None").
31
+ Splitting `authors` into a name-only entry plus an email-only entry,
32
+ and adding a parallel `maintainers` field, makes the package author
33
+ display correctly across PyPI, Libraries.io, and pypistats.
34
+
35
+ ## [0.2.0] — 2026-04-26
36
+
37
+ ### Added
38
+
39
+ - **SDK adapters** for the two dominant agent toolchains, eliminating the need
40
+ for manual `Trace` instrumentation:
41
+ - `agentprdiff.adapters.openai.instrument_client` — context manager that
42
+ monkey-patches `client.chat.completions.create` for the duration of one
43
+ agent call. Records each invocation as an `LLMCall` (provider, model,
44
+ input messages, output text, tool calls, tokens, cost, latency) and
45
+ restores the original on exit. Works with **OpenAI, Groq, Gemini's
46
+ OpenAI-compatible endpoint, OpenRouter, Ollama, vLLM, Together,
47
+ Fireworks, DeepInfra**, and any other SDK that follows the OpenAI client
48
+ shape.
49
+ - `agentprdiff.adapters.anthropic.instrument_client` — equivalent for the
50
+ Anthropic Messages API (`client.messages.create`). Handles the
51
+ content-block response shape (text + `tool_use` blocks) and the
52
+ Messages-API token field names.
53
+ - `instrument_tools(tool_map, trace)` — wraps a dict of callables so each
54
+ invocation records a `ToolCall` with name, arguments, result, latency,
55
+ and any raised exception. Shared between both adapters.
56
+ - `agentprdiff.adapters.pricing` — curated model→price table for cost
57
+ estimation, with `register_prices()` and per-call `prices=` overrides.
58
+ Unknown models record `cost_usd=0.0` and emit a single `RuntimeWarning`
59
+ per process so missing pricing is loud rather than silent.
60
+ - Documentation: `docs/adapters.md` (full reference) and
61
+ `docs/adapters-vercel.md` (manual integration recipe for the Vercel AI
62
+ SDK, which is JS-only and lives in a future companion package).
63
+ - `AGENTS.md` at the repo root — an instruction set written for AI
64
+ coding agents (Claude Code, Cursor, Aider, etc.) that have been asked
65
+ to add `agentprdiff` to a codebase. Covers codebase discovery,
66
+ contract identification, wrap-the-agent recipes (OpenAI / Anthropic /
67
+ custom), stub patterns, suite scaffolding, baseline recording, CI
68
+ wiring, common pitfalls, and a validation checklist. Optimized for
69
+ AI-agent-driven adoption with copy-paste templates.
70
+ - `docs/ai-driven-adoption.md` — human-facing companion to AGENTS.md.
71
+ Three prompt templates (minimum viable / recommended / contract-driven)
72
+ for adopters using Claude Code / Cursor / Aider, plus a sample
73
+ first-session transcript and tips for working with the AI agent
74
+ through the adoption flow.
75
+ - `docs/suite-layout.md` — canonical reference for the suite directory
76
+ structure. Lists each file (`suites/<project>.py`, `_eval_agent.py`,
77
+ `_stubs.py`, baselines, CI workflow, etc.), classifies them as
78
+ mandatory / recommended / optional, and specifies what each must
79
+ and must not contain. Cross-referenced from AGENTS.md and the
80
+ validation checklist.
81
+
82
+ ### Changed
83
+
84
+ - The suite loader now inserts the current working directory onto
85
+ `sys.path` in addition to the suite file's parent directory. Adopters
86
+ who run `agentprdiff record suites/foo.py` from their project root no
87
+ longer have to manually patch `sys.path` to import their own modules
88
+ (e.g. `from agent.agent import ...`, `from config import ...`).
89
+ Both insertions are reverted after the suite loads, so no path leakage
90
+ between runs.
91
+
92
+ ### Notes
93
+
94
+ - The base `pip install agentprdiff` does **not** require the `openai` or
95
+ `anthropic` packages. The adapters operate on a client object's shape,
96
+ not on imported SDK modules — so installing only the SDKs you actually
97
+ use keeps the dependency footprint small. Optional extras are still
98
+ declared (`agentprdiff[openai]`, `agentprdiff[anthropic]`) for adopters
99
+ who prefer to pin the SDK version alongside agentprdiff itself.
100
+
101
+ ## [0.1.0] — 2026-04-22
102
+
103
+ Initial public release.
104
+
105
+ ### Added
106
+
107
+ - Core `Suite` / `Case` / `Trace` model for defining agent regression tests.
108
+ - Deterministic graders: `contains`, `contains_any`, `regex_match`, `tool_called`,
109
+ `tool_sequence`, `output_length_lt`, `latency_lt_ms`, `cost_lt_usd`,
110
+ `no_tool_called`.
111
+ - Semantic grader (`semantic`) with a pluggable `judge` callable and built-in
112
+ fake judge for CI environments without API keys.
113
+ - Baseline store (JSON files under `.agentprdiff/baselines/`) designed to be
114
+ committed to version control.
115
+ - Trace diff engine producing a structured `TraceDelta` (assertion pass/fail
116
+ changes, cost delta, latency delta, tool-call sequence changes, output
117
+ change).
118
+ - CLI: `agentprdiff init`, `agentprdiff record`, `agentprdiff check`, `agentprdiff diff`.
119
+ - Rich-formatted terminal reporter and machine-readable JSON reporter for CI.
120
+ - Quickstart example with a mock agent that runs without any API keys.
121
+ - Pytest test suite covering graders, runner, differ, store, and CLI smoke.
122
+ - GitHub Actions CI workflow.
123
+
124
+ ### Known limitations
125
+
126
+ - Only a manual instrumentation API for provider SDKs is shipped in 0.1.0.
127
+ Drop-in wrappers for OpenAI / Anthropic / Vercel AI SDK are planned for 0.2.
128
+ - The semantic grader's built-in judge supports OpenAI and Anthropic via user-
129
+ supplied API keys; hosted judge endpoints are not yet offered.
@@ -1,12 +1,14 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: agentprdiff
3
- Version: 0.1.0
3
+ Version: 0.2.1
4
4
  Summary: Guard your LLM agents in CI. Snapshot tests that catch behavioral regressions when models, prompts, or vendors change.
5
5
  Project-URL: Homepage, https://github.com/vnageshwaran-de/agentprdiff
6
6
  Project-URL: Documentation, https://github.com/vnageshwaran-de/agentprdiff#readme
7
7
  Project-URL: Issues, https://github.com/vnageshwaran-de/agentprdiff/issues
8
8
  Project-URL: Repository, https://github.com/vnageshwaran-de/agentprdiff
9
- Author-email: Vinoth Nageshwaran <vnageshwaran@gmail.com>
9
+ Author: Vinoth Nageshwaran
10
+ Author-email: vnageshwaran@gmail.com
11
+ Maintainer-email: Vinoth Nageshwaran <vnageshwaran@gmail.com>
10
12
  License: MIT
11
13
  License-File: LICENSE
12
14
  Keywords: agent,ci,evaluation,llm,observability,regression-testing,snapshot-testing
@@ -48,7 +50,9 @@ pip install agentprdiff
48
50
  [![CI](https://github.com/vnageshwaran-de/agentprdiff/actions/workflows/ci.yml/badge.svg)](https://github.com/vnageshwaran-de/agentprdiff/actions/workflows/ci.yml)
49
51
  [![PyPI](https://img.shields.io/pypi/v/agentprdiff.svg)](https://pypi.org/project/agentprdiff/)
50
52
  [![Python](https://img.shields.io/pypi/pyversions/agentprdiff.svg)](https://pypi.org/project/agentprdiff/)
51
- [![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](./LICENSE)
53
+ [![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://github.com/vnageshwaran-de/agentprdiff/blob/main/LICENSE)
54
+
55
+ > **Adopting with an AI coding agent?** Point Claude Code, Cursor, Aider, or any agentic IDE at [`AGENTS.md`](https://github.com/vnageshwaran-de/agentprdiff/blob/main/AGENTS.md) — a step-by-step adoption playbook the agent reads directly. Humans driving the adoption: see [`docs/ai-driven-adoption.md`](https://github.com/vnageshwaran-de/agentprdiff/blob/main/docs/ai-driven-adoption.md) for copy-paste prompt templates. The canonical file layout — what's mandatory, what's recommended, what's optional — is at [`docs/suite-layout.md`](https://github.com/vnageshwaran-de/agentprdiff/blob/main/docs/suite-layout.md).
52
56
 
53
57
  ## Why
54
58
 
@@ -100,6 +104,7 @@ That's the whole product. Four CLI commands. One Python file. Zero framework loc
100
104
  - **Diff engine** — per-case `TraceDelta` with assertion pass/fail changes, cost delta, latency delta, tool-sequence changes, and a unified output diff.
101
105
  - **CI-ready CLI** — exit 1 on regression, `--json-out` for artifact archiving, Rich-formatted terminal output.
102
106
  - **Zero SDK lock-in** — works with OpenAI, Anthropic, Gemini, Bedrock, LangChain, LangGraph, LlamaIndex, Vercel AI SDK, custom wrappers — if you can wrap your agent in a function, `agentprdiff` can test it.
107
+ - **One-line SDK adapters** — `with instrument_client(client) as trace:` automatically records every LLM and tool call when you're on the OpenAI Python SDK (or any OpenAI-compatible provider — Groq / Gemini / OpenRouter / Ollama / vLLM) or the Anthropic SDK. No manual `Trace` wiring required.
103
108
 
104
109
  ## How it compares
105
110
 
@@ -124,7 +129,34 @@ This is the same loop as Jest snapshot tests or VCR cassettes — applied to LLM
124
129
 
125
130
  ## Instrumenting your agent
126
131
 
127
- `agentprdiff` doesn't monkey-patch anything. Your agent returns `(output, Trace)`:
132
+ You have two paths. Most agents need the first.
133
+
134
+ ### Option A — SDK adapters (zero manual work)
135
+
136
+ If your agent uses the OpenAI Python SDK (or any OpenAI-compatible provider — Groq, Gemini, OpenRouter, Ollama, vLLM, Together, Fireworks, DeepInfra) or the Anthropic SDK, the SDK adapter captures every model and tool call automatically:
137
+
138
+ ```python
139
+ from openai import OpenAI
140
+ from agentprdiff.adapters.openai import instrument_client, instrument_tools
141
+
142
+ TOOL_MAP = {"lookup_order": lookup_order, "send_email": send_email}
143
+
144
+ def my_agent(query: str):
145
+ client = OpenAI()
146
+ with instrument_client(client) as trace:
147
+ tools = instrument_tools(TOOL_MAP, trace)
148
+ # ... your existing tool-calling loop, untouched ...
149
+ # the only swap: TOOL_MAP[fn](**args) → tools[fn](**args)
150
+ return final_text, trace
151
+ ```
152
+
153
+ The patch is scoped to the specific client instance and reversed when the `with` block exits — no global SDK state is touched. Anthropic adopters use `agentprdiff.adapters.anthropic` with the same shape.
154
+
155
+ See [`docs/adapters.md`](https://github.com/vnageshwaran-de/agentprdiff/blob/main/docs/adapters.md) for the full reference, including pricing overrides, custom provider tags, and recipes for nested agents.
156
+
157
+ ### Option B — Manual instrumentation
158
+
159
+ If you're not on either SDK, or you want full control, build the `Trace` yourself — `agentprdiff` doesn't require any monkey-patching:
128
160
 
129
161
  ```python
130
162
  from agentprdiff import Trace, LLMCall, ToolCall
@@ -168,7 +200,7 @@ jobs:
168
200
  with: { name: agentprdiff, path: artifacts/ }
169
201
  ```
170
202
 
171
- See [`docs/ci-integration.md`](./docs/ci-integration.md) for GitLab, CircleCI, and Buildkite.
203
+ See [`docs/ci-integration.md`](https://github.com/vnageshwaran-de/agentprdiff/blob/main/docs/ci-integration.md) for GitLab, CircleCI, and Buildkite.
172
204
 
173
205
  ## Quickstart
174
206
 
@@ -191,10 +223,10 @@ agentprdiff check suite.py # exit 1; see the diff
191
223
 
192
224
  ## Status
193
225
 
194
- `agentprdiff` is **alpha** (0.1.0). The core model and CLI are stable; provider-specific SDK wrappers and a LangChain/LangGraph integration are on the 0.2 roadmap. See [`CHANGELOG.md`](./CHANGELOG.md).
226
+ `agentprdiff` is **alpha** (0.2.x). The core model, CLI, and OpenAI / Anthropic SDK adapters are stable. LangChain/LangGraph adapters and a JS companion package for the Vercel AI SDK are on the 0.3 roadmap. See [`CHANGELOG.md`](https://github.com/vnageshwaran-de/agentprdiff/blob/main/CHANGELOG.md).
195
227
 
196
228
  Feedback, bug reports, and PRs extremely welcome. Open an issue or @ me.
197
229
 
198
230
  ## License
199
231
 
200
- MIT. See [`LICENSE`](./LICENSE).
232
+ MIT. See [`LICENSE`](https://github.com/vnageshwaran-de/agentprdiff/blob/main/LICENSE).
@@ -11,7 +11,9 @@ pip install agentprdiff
11
11
  [![CI](https://github.com/vnageshwaran-de/agentprdiff/actions/workflows/ci.yml/badge.svg)](https://github.com/vnageshwaran-de/agentprdiff/actions/workflows/ci.yml)
12
12
  [![PyPI](https://img.shields.io/pypi/v/agentprdiff.svg)](https://pypi.org/project/agentprdiff/)
13
13
  [![Python](https://img.shields.io/pypi/pyversions/agentprdiff.svg)](https://pypi.org/project/agentprdiff/)
14
- [![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](./LICENSE)
14
+ [![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://github.com/vnageshwaran-de/agentprdiff/blob/main/LICENSE)
15
+
16
+ > **Adopting with an AI coding agent?** Point Claude Code, Cursor, Aider, or any agentic IDE at [`AGENTS.md`](https://github.com/vnageshwaran-de/agentprdiff/blob/main/AGENTS.md) — a step-by-step adoption playbook the agent reads directly. Humans driving the adoption: see [`docs/ai-driven-adoption.md`](https://github.com/vnageshwaran-de/agentprdiff/blob/main/docs/ai-driven-adoption.md) for copy-paste prompt templates. The canonical file layout — what's mandatory, what's recommended, what's optional — is at [`docs/suite-layout.md`](https://github.com/vnageshwaran-de/agentprdiff/blob/main/docs/suite-layout.md).
15
17
 
16
18
  ## Why
17
19
 
@@ -63,6 +65,7 @@ That's the whole product. Four CLI commands. One Python file. Zero framework loc
63
65
  - **Diff engine** — per-case `TraceDelta` with assertion pass/fail changes, cost delta, latency delta, tool-sequence changes, and a unified output diff.
64
66
  - **CI-ready CLI** — exit 1 on regression, `--json-out` for artifact archiving, Rich-formatted terminal output.
65
67
  - **Zero SDK lock-in** — works with OpenAI, Anthropic, Gemini, Bedrock, LangChain, LangGraph, LlamaIndex, Vercel AI SDK, custom wrappers — if you can wrap your agent in a function, `agentprdiff` can test it.
68
+ - **One-line SDK adapters** — `with instrument_client(client) as trace:` automatically records every LLM and tool call when you're on the OpenAI Python SDK (or any OpenAI-compatible provider — Groq / Gemini / OpenRouter / Ollama / vLLM) or the Anthropic SDK. No manual `Trace` wiring required.
66
69
 
67
70
  ## How it compares
68
71
 
@@ -87,7 +90,34 @@ This is the same loop as Jest snapshot tests or VCR cassettes — applied to LLM
87
90
 
88
91
  ## Instrumenting your agent
89
92
 
90
- `agentprdiff` doesn't monkey-patch anything. Your agent returns `(output, Trace)`:
93
+ You have two paths. Most agents need the first.
94
+
95
+ ### Option A — SDK adapters (zero manual work)
96
+
97
+ If your agent uses the OpenAI Python SDK (or any OpenAI-compatible provider — Groq, Gemini, OpenRouter, Ollama, vLLM, Together, Fireworks, DeepInfra) or the Anthropic SDK, the SDK adapter captures every model and tool call automatically:
98
+
99
+ ```python
100
+ from openai import OpenAI
101
+ from agentprdiff.adapters.openai import instrument_client, instrument_tools
102
+
103
+ TOOL_MAP = {"lookup_order": lookup_order, "send_email": send_email}
104
+
105
+ def my_agent(query: str):
106
+ client = OpenAI()
107
+ with instrument_client(client) as trace:
108
+ tools = instrument_tools(TOOL_MAP, trace)
109
+ # ... your existing tool-calling loop, untouched ...
110
+ # the only swap: TOOL_MAP[fn](**args) → tools[fn](**args)
111
+ return final_text, trace
112
+ ```
113
+
114
+ The patch is scoped to the specific client instance and reversed when the `with` block exits — no global SDK state is touched. Anthropic adopters use `agentprdiff.adapters.anthropic` with the same shape.
115
+
116
+ See [`docs/adapters.md`](https://github.com/vnageshwaran-de/agentprdiff/blob/main/docs/adapters.md) for the full reference, including pricing overrides, custom provider tags, and recipes for nested agents.
117
+
118
+ ### Option B — Manual instrumentation
119
+
120
+ If you're not on either SDK, or you want full control, build the `Trace` yourself — `agentprdiff` doesn't require any monkey-patching:
91
121
 
92
122
  ```python
93
123
  from agentprdiff import Trace, LLMCall, ToolCall
@@ -131,7 +161,7 @@ jobs:
131
161
  with: { name: agentprdiff, path: artifacts/ }
132
162
  ```
133
163
 
134
- See [`docs/ci-integration.md`](./docs/ci-integration.md) for GitLab, CircleCI, and Buildkite.
164
+ See [`docs/ci-integration.md`](https://github.com/vnageshwaran-de/agentprdiff/blob/main/docs/ci-integration.md) for GitLab, CircleCI, and Buildkite.
135
165
 
136
166
  ## Quickstart
137
167
 
@@ -154,10 +184,10 @@ agentprdiff check suite.py # exit 1; see the diff
154
184
 
155
185
  ## Status
156
186
 
157
- `agentprdiff` is **alpha** (0.1.0). The core model and CLI are stable; provider-specific SDK wrappers and a LangChain/LangGraph integration are on the 0.2 roadmap. See [`CHANGELOG.md`](./CHANGELOG.md).
187
+ `agentprdiff` is **alpha** (0.2.x). The core model, CLI, and OpenAI / Anthropic SDK adapters are stable. LangChain/LangGraph adapters and a JS companion package for the Vercel AI SDK are on the 0.3 roadmap. See [`CHANGELOG.md`](https://github.com/vnageshwaran-de/agentprdiff/blob/main/CHANGELOG.md).
158
188
 
159
189
  Feedback, bug reports, and PRs extremely welcome. Open an issue or @ me.
160
190
 
161
191
  ## License
162
192
 
163
- MIT. See [`LICENSE`](./LICENSE).
193
+ MIT. See [`LICENSE`](https://github.com/vnageshwaran-de/agentprdiff/blob/main/LICENSE).
@@ -0,0 +1,140 @@
1
+ # Regression tour
2
+
3
+ A complete walkthrough of every grader and every failure mode in `agentprdiff`. Runs without API keys (uses `fake_judge` for the semantic grader so no OpenAI/Anthropic key is required).
4
+
5
+ ## What this exercises
6
+
7
+ - All 10 deterministic and semantic graders: `contains`, `contains_any`, `regex_match`, `tool_called`, `tool_sequence`, `no_tool_called`, `output_length_lt`, `latency_lt_ms`, `cost_lt_usd`, `semantic`.
8
+ - All 6 regression scenarios the differ can detect: output drift, extra tool, missing tool, tool reordering, latency regression, cost regression.
9
+ - The Rich terminal reporter and exit-code behavior used in CI.
10
+
11
+ ## A note on invocation
12
+
13
+ The commands below use `agentprdiff` directly. If pip's user-script directory isn't on your PATH, substitute `python3 -m agentprdiff.cli` everywhere — both forms are equivalent.
14
+
15
+ ## Setup (one-time)
16
+
17
+ ```bash
18
+ cd examples/regression-tour
19
+ agentprdiff init # creates the .agentprdiff/ scaffolding
20
+ agentprdiff record suite.py # captures the baseline trace
21
+ ```
22
+
23
+ You should see baselines written under `.agentprdiff/baselines/`. In a real project these get committed to git — that's the whole point of agentprdiff.
24
+
25
+ ## Happy path
26
+
27
+ ```bash
28
+ agentprdiff check suite.py
29
+ echo "exit: $?" # 0
30
+ ```
31
+
32
+ All three cases pass with no diff against the recorded baseline.
33
+
34
+ ## Regression scenarios
35
+
36
+ Each command below injects one specific regression by setting `MODE`. Every one should fail with a clear diff and a non-zero exit code.
37
+
38
+ ### 1. Output text drifted
39
+
40
+ ```bash
41
+ MODE=output_changed agentprdiff check suite.py
42
+ echo "exit: $?" # non-zero
43
+ ```
44
+
45
+ The agent's refund response changes from the baseline phrasing to "Refund initiated. Please allow 7–10 business days for processing." That trips:
46
+
47
+ - `contains("refund")` — still passes (the new text mentions refund)
48
+ - `contains_any(["business days", "card", "processed"])` — still passes
49
+ - `regex_match(r"\$\d+\.\d{2}")` — **fails** (no dollar amount in the new output)
50
+ - `output_length_lt(500)` — still passes
51
+ - `semantic(...)` — likely **fails** depending on judge backend
52
+
53
+ The terminal reporter prints a unified output diff so the reviewer can see exactly what changed.
54
+
55
+ ### 2. Extra tool call
56
+
57
+ ```bash
58
+ MODE=tool_added agentprdiff check suite.py
59
+ echo "exit: $?" # non-zero
60
+ ```
61
+
62
+ The agent calls `check_inventory` after `lookup_order`. That trips:
63
+
64
+ - `tool_sequence(["lookup_order"])` — **fails** (sequence is now `["lookup_order", "check_inventory"]`)
65
+ - `no_tool_called("check_inventory")` — **fails**
66
+
67
+ ### 3. Missing tool call
68
+
69
+ ```bash
70
+ MODE=tool_removed agentprdiff check suite.py
71
+ echo "exit: $?" # non-zero
72
+ ```
73
+
74
+ The agent never calls `lookup_order` and produces a fallback "trouble looking up your order" response. That trips:
75
+
76
+ - `tool_called("lookup_order")` — **fails**
77
+ - `tool_sequence(["lookup_order"])` — **fails**
78
+ - `contains("refund")` — **fails** (output text changed)
79
+ - `regex_match(r"\$\d+\.\d{2}")` — **fails**
80
+
81
+ ### 4. Tool order swapped
82
+
83
+ ```bash
84
+ MODE=tool_reordered agentprdiff check suite.py
85
+ echo "exit: $?" # non-zero
86
+ ```
87
+
88
+ The agent calls `check_inventory` *before* `lookup_order`. Same tools, wrong order:
89
+
90
+ - `tool_sequence(["lookup_order"])` — **fails** (sequence is `["check_inventory", "lookup_order"]`)
91
+ - `no_tool_called("check_inventory")` — **fails**
92
+
93
+ ### 5. Latency regression
94
+
95
+ ```bash
96
+ MODE=latency_regressed agentprdiff check suite.py
97
+ echo "exit: $?" # non-zero
98
+ ```
99
+
100
+ Planner LLM call jumps from 180 ms to 8 s, blowing past the 5 s cap:
101
+
102
+ - `latency_lt_ms(5_000)` — **fails**
103
+
104
+ The reporter shows the latency delta against baseline.
105
+
106
+ ### 6. Cost regression
107
+
108
+ ```bash
109
+ MODE=cost_regressed agentprdiff check suite.py
110
+ echo "exit: $?" # non-zero
111
+ ```
112
+
113
+ Responder cost jumps from $0.0008 to $0.10 per call:
114
+
115
+ - `cost_lt_usd(0.01)` — **fails**
116
+
117
+ The reporter shows the cost delta against baseline.
118
+
119
+ ## Run them all back to back
120
+
121
+ ```bash
122
+ ./tour.sh
123
+ ```
124
+
125
+ Runs every scenario with banners, prints exit codes, and gives you a one-screen overview of what the tool detects.
126
+
127
+ ## Resetting the baseline
128
+
129
+ If you want to make the new behavior the new baseline (e.g., you intentionally changed the agent), re-record:
130
+
131
+ ```bash
132
+ MODE=output_changed agentprdiff record suite.py # baseline now matches the changed output
133
+ agentprdiff check suite.py # passes against the new baseline
134
+ ```
135
+
136
+ This is the workflow you'll use in real projects when a model upgrade or prompt change is intentional.
137
+
138
+ ## Why this matters
139
+
140
+ The point of `agentprdiff` is that every one of these scenarios should be caught **before** the change reaches production. Run `agentprdiff check` in CI on every PR and the merge is blocked when behavior changes — same as `pytest` blocks merges when tests break.
@@ -4,12 +4,16 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "agentprdiff"
7
- version = "0.1.0"
7
+ version = "0.2.1"
8
8
  description = "Guard your LLM agents in CI. Snapshot tests that catch behavioral regressions when models, prompts, or vendors change."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
11
11
  license = { text = "MIT" }
12
- authors = [{ name = "Vinoth Nageshwaran", email = "vnageshwaran@gmail.com" }]
12
+ authors = [
13
+ { name = "Vinoth Nageshwaran" },
14
+ { email = "vnageshwaran@gmail.com" },
15
+ ]
16
+ maintainers = [{ name = "Vinoth Nageshwaran", email = "vnageshwaran@gmail.com" }]
13
17
  keywords = [
14
18
  "llm",
15
19
  "agent",
@@ -30,6 +30,22 @@ Run from the shell::
30
30
  agentprdiff init
31
31
  agentprdiff record path/to/my_suite.py # save baselines
32
32
  agentprdiff check path/to/my_suite.py # diff against baselines; exit 1 on regression
33
+
34
+ If your agent already uses the OpenAI Python SDK (or any OpenAI-compatible
35
+ provider — Groq, Gemini, OpenRouter, Ollama, vLLM) or the Anthropic SDK, the
36
+ SDK adapters capture every model and tool call automatically, no manual Trace
37
+ wiring required::
38
+
39
+ from agentprdiff.adapters.openai import instrument_client, instrument_tools
40
+
41
+ def my_agent(query):
42
+ client = OpenAI(...)
43
+ with instrument_client(client) as trace:
44
+ tools = instrument_tools(TOOL_MAP, trace)
45
+ # ... your existing tool-calling loop, untouched ...
46
+ return final_text, trace
47
+
48
+ See ``docs/adapters.md`` for the full reference.
33
49
  """
34
50
 
35
51
  from __future__ import annotations
@@ -51,7 +67,7 @@ from .differ import AssertionChange, TraceDelta, diff_traces
51
67
  from .runner import CaseReport, Runner, RunReport
52
68
  from .store import BaselineStore
53
69
 
54
- __version__ = "0.1.0"
70
+ __version__ = "0.2.1"
55
71
 
56
72
  __all__ = [
57
73
  # core
@@ -0,0 +1,43 @@
1
+ """SDK adapters for agentprdiff.
2
+
3
+ The adapters take an agent that uses a real LLM SDK (OpenAI, Anthropic, or any
4
+ OpenAI-compatible provider like Groq, Gemini's openai-compat endpoint,
5
+ OpenRouter, Ollama, or vLLM) and capture every model call as an `LLMCall` on a
6
+ `Trace` — without forcing the user to rewrite their agent loop.
7
+
8
+ The pattern is::
9
+
10
+ from agentprdiff.adapters.openai import instrument_client, instrument_tools
11
+
12
+ def my_agent(query: str):
13
+ client = OpenAI(...)
14
+ with instrument_client(client) as trace:
15
+ tools = instrument_tools(TOOL_MAP, trace)
16
+ # ... user's existing tool-calling loop, untouched ...
17
+ return final_text, trace
18
+
19
+ Submodules are imported lazily so the base `agentprdiff` install doesn't pull
20
+ in `openai` / `anthropic` unless the user opts in via the extras::
21
+
22
+ pip install "agentprdiff[openai]"
23
+ pip install "agentprdiff[anthropic]"
24
+
25
+ See `docs/adapters.md` for the full reference.
26
+ """
27
+
28
+ from __future__ import annotations
29
+
30
+ # Re-export pricing helpers — these are pure-Python and have no SDK dependency.
31
+ from .pricing import (
32
+ DEFAULT_PRICES,
33
+ PriceTable,
34
+ estimate_cost_usd,
35
+ register_prices,
36
+ )
37
+
38
+ __all__ = [
39
+ "DEFAULT_PRICES",
40
+ "PriceTable",
41
+ "estimate_cost_usd",
42
+ "register_prices",
43
+ ]