brooder 0.1.0__py3-none-any.whl

brooder/storage.py

@@ -0,0 +1,150 @@
+"""On-disk storage for baselines and runs.
+
+Baselines are golden runs, committed to the repo so behavioral changes show up in code review.
+Runs are the latest (uncommitted) executions used to compute diffs. Writes are atomic and
+corrupt records raise :class:`~brooder.errors.CorruptRecordError` rather than crashing.
+
+    .brooder/
+      baselines/<agent>/<case_id>.json   # golden, committed
+      runs/<agent>/<case_id>.json        # latest run, gitignored
+"""
+
+from __future__ import annotations
+
+import os
+import tempfile
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Optional
+
+from pydantic import ValidationError
+
+from .errors import CorruptRecordError
+from .models import Run
+
+BROODER_DIR = ".brooder"
+
+
+def _root(base: Optional[Path] = None) -> Path:
+    return (base or Path.cwd()) / BROODER_DIR
+
+
+def baseline_path(agent: str, case_id: str, base: Optional[Path] = None) -> Path:
+    """Return the file path of a case's golden baseline.
+
+    Args:
+        agent: The agent name.
+        case_id: The case id (see :func:`~brooder.models.make_case_id`).
+        base: Project root; defaults to the current working directory.
+
+    Returns:
+        The path to ``.brooder/baselines/<agent>/<case_id>.json``.
+    """
+    return _root(base) / "baselines" / agent / f"{case_id}.json"
+
+
+def run_path(agent: str, case_id: str, base: Optional[Path] = None) -> Path:
+    """Return the file path of a case's latest run.
+
+    Args:
+        agent: The agent name.
+        case_id: The case id.
+        base: Project root; defaults to the current working directory.
+
+    Returns:
+        The path to ``.brooder/runs/<agent>/<case_id>.json``.
+    """
+    return _root(base) / "runs" / agent / f"{case_id}.json"
+
+
+def _atomic_write(path: Path, text: str) -> None:
+    """Write ``text`` to ``path`` atomically (temp file + ``os.replace``)."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, tmp = tempfile.mkstemp(dir=str(path.parent), suffix=".tmp")
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as handle:
+            handle.write(text)
+        os.replace(tmp, path)
+    finally:
+        if os.path.exists(tmp):
+            os.remove(tmp)
+
+
+def _load(path: Path) -> Run:
+    try:
+        return Run.model_validate_json(path.read_text(encoding="utf-8"))
+    except (ValidationError, ValueError, OSError) as exc:
+        raise CorruptRecordError(f"{path} is not a valid Brooder record: {exc}") from exc
+
+
+def save_baseline(run: Run, base: Optional[Path] = None) -> None:
+    """Persist ``run`` as the golden baseline for its case.
+
+    Args:
+        run: The run to store as the baseline.
+        base: Project root; defaults to the current working directory.
+    """
+    _atomic_write(baseline_path(run.agent, run.case_id, base), run.model_dump_json(indent=2))
+
+
+def load_baseline(agent: str, case_id: str, base: Optional[Path] = None) -> Optional[Run]:
+    """Load a case's golden baseline, if one exists.
+
+    Args:
+        agent: The agent name.
+        case_id: The case id.
+        base: Project root; defaults to the current working directory.
+
+    Returns:
+        The stored :class:`~brooder.models.Run`, or ``None`` if there is no baseline.
+
+    Raises:
+        CorruptRecordError: If the baseline file exists but cannot be parsed.
+    """
+    path = baseline_path(agent, case_id, base)
+    return _load(path) if path.exists() else None
+
+
+def save_run(run: Run, base: Optional[Path] = None) -> None:
+    """Persist ``run`` as the latest run for its case.
+
+    Args:
+        run: The run to store.
+        base: Project root; defaults to the current working directory.
+    """
+    _atomic_write(run_path(run.agent, run.case_id, base), run.model_dump_json(indent=2))
+
+
+def iter_runs(base: Optional[Path] = None) -> Iterator[Run]:
+    """Yield every stored latest-run record.
+
+    Args:
+        base: Project root; defaults to the current working directory.
+
+    Yields:
+        Each stored :class:`~brooder.models.Run`, sorted by path.
+
+    Raises:
+        CorruptRecordError: If a run file cannot be parsed.
+    """
+    runs_dir = _root(base) / "runs"
+    if not runs_dir.exists():
+        return
+    for file in sorted(runs_dir.rglob("*.json")):
+        yield _load(file)
+
+
+def promote_runs_to_baselines(base: Optional[Path] = None) -> int:
+    """Accept every latest run as the new baseline (like ``jest -u``).
+
+    Args:
+        base: Project root; defaults to the current working directory.
+
+    Returns:
+        The number of runs promoted to baselines.
+    """
+    count = 0
+    for run in iter_runs(base):
+        save_baseline(run, base)
+        count += 1
+    return count

brooder-0.1.0.dist-info/METADATA

@@ -0,0 +1,338 @@
+Metadata-Version: 2.4
+Name: brooder
+Version: 0.1.0
+Summary: Snapshot testing for AI agents — catch behavior regressions before they ship.
+Project-URL: Homepage, https://brooder.dev
+Project-URL: Repository, https://github.com/agentbrooder/brooder
+Project-URL: Issues, https://github.com/agentbrooder/brooder/issues
+Author: Brooder
+License: Apache-2.0
+License-File: LICENSE
+License-File: NOTICE
+Keywords: agents,ai,ci,evals,llm,regression,snapshot,testing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Testing
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: pydantic>=2.5
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: rich>=13.0
+Requires-Dist: typer>=0.12
+Provides-Extra: claude-agent
+Requires-Dist: claude-agent-sdk>=0.1; extra == 'claude-agent'
+Provides-Extra: dev
+Requires-Dist: mypy>=1.11; extra == 'dev'
+Requires-Dist: pre-commit>=3.7; extra == 'dev'
+Requires-Dist: pytest-cov>=4.1; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Requires-Dist: types-pyyaml>=6.0; extra == 'dev'
+Provides-Extra: docs
+Requires-Dist: mkdocs-material>=9.5; extra == 'docs'
+Requires-Dist: mkdocstrings[python]>=0.25; extra == 'docs'
+Provides-Extra: langchain
+Requires-Dist: langchain-core>=0.3; extra == 'langchain'
+Provides-Extra: openai-agents
+Requires-Dist: openai-agents>=0.1; extra == 'openai-agents'
+Provides-Extra: otel
+Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.20; extra == 'otel'
+Requires-Dist: opentelemetry-sdk>=1.20; extra == 'otel'
+Description-Content-Type: text/markdown
+
+<p align="center">
+  <img src="assets/banner.svg" alt="Brooder — snapshot testing for AI agents" width="760">
+</p>
+
+<p align="center">
+  <a href="https://github.com/agentbrooder/brooder/actions/workflows/ci.yml"><img src="https://github.com/agentbrooder/brooder/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
+  <a href="https://pypi.org/project/brooder/"><img src="https://img.shields.io/pypi/v/brooder?color=3b82f6" alt="PyPI"></a>
+  <a href="https://pypi.org/project/brooder/"><img src="https://img.shields.io/pypi/pyversions/brooder" alt="Python versions"></a>
+  <a href="LICENSE"><img src="https://img.shields.io/badge/license-Apache--2.0-blue" alt="License: Apache-2.0"></a>
+  <a href="https://github.com/astral-sh/ruff"><img src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json" alt="Ruff"></a>
+</p>
+
+**Snapshot testing for AI agents. Catch behavior regressions before they ship.**
+
+Your AI agent is one model upgrade away from silently breaking. You bump the model, tweak a
+prompt, or change a tool — and the agent starts behaving differently. You find out from a customer.
+
+Brooder is the safety net. Wrap your agent once, and Brooder records its real runs as **golden
+baselines**. Every time you change the model, a prompt, or a tool, it re-runs and shows you a
+**behavioral diff** — what changed, what broke — and fails your CI if it regressed.
+
+No eval datasets to hand-write. One command. It's `jest --updateSnapshot`, but for agents.
+
+```bash
+pip install brooder
+```
+
+<p align="center">
+  <img src="assets/demo.svg" alt="brooder migrate catching a dropped tool call and a flipped answer" width="760">
+</p>
+
+> Status: early alpha, built in public. Apache-2.0.
+
+---
+
+## 60-second demo (no API keys needed)
+
+The included example agent simulates a model upgrade with an env var, so you can see Brooder catch
+a real regression completely offline.
+
+```bash
+git clone https://github.com/agentbrooder/brooder && cd brooder
+pip install -e .
+
+# The signature move: what breaks if I migrate from one model to another?
+brooder migrate --from gpt-4o --to gpt-5-new examples/regressing_agent.py
+```
+
+Output (abridged):
+
+```
+──────────────────────── Model Migration Report ────────────────────────
+ 1 of 3 cases change behavior when migrating gpt-4o → gpt-5-new.
+
+ support-agent · e1ded4070eee · REGRESSED · stability 40
+   path diverged at step 0: was TOOL create_ticket(order=12345), now dropped
+   - trajectory[0]  {'name': 'create_ticket', 'args': {'order': '12345'}}
+   ~ output
+       before: I've started your refund.
+       after:  Refunds are not supported.
+```
+
+The "new model" silently stopped creating the refund ticket **and** flipped its answer. That would
+have shipped to production unnoticed. Brooder caught it — and exited non-zero, so CI would block it.
+
+---
+
+## The normal workflow
+
+```bash
+brooder record examples/regressing_agent.py     # capture golden baselines from real runs
+brooder run    examples/regressing_agent.py     # re-run after a change, diff vs baseline
+brooder diff                                    # see exactly what changed
+brooder approve                                 # accept the new behavior as the baseline
+```
+
+`brooder run` exits non-zero when behavior regressed — drop it into CI and it gates your PRs.
+
+---
+
+## Instrument your own agent
+
+Add one decorator. Log tool calls with one function. That's the whole SDK.
+
+```python
+import brooder
+
+def search_kb(query):
+    brooder.tool_call("search_kb", {"query": query}, result="...")
+    return "..."
+
+@brooder.record("support-agent")
+def agent(question: str) -> str:
+    docs = search_kb(question)
+    return answer_from(docs)
+
+# call it over your real inputs; brooder records/replays automatically
+```
+
+Then run it through the CLI. Baselines are plain JSON committed to your repo, so diffs show up in
+code review like any other change.
+
+---
+
+## Auto-capture (no manual `tool_call`)
+
+Wrap your LLM client and Brooder records the model's tool-call decisions automatically:
+
+```python
+import brooder
+import openai
+
+client = brooder.instrument(openai.OpenAI())
+# now every client.chat.completions.create(...) call is captured while recording
+```
+
+Supported providers: **OpenAI**, **Azure OpenAI**, **Anthropic**, **AWS Bedrock**, and
+**Google (Gemini / Vertex)**. The provider is auto-detected; override it with
+`brooder.instrument(client, provider="bedrock")`. Model *names* are intentionally not diffed, so
+switching models isn't itself a change — only the model's *behavior* (which tools it calls, with
+what arguments) is.
+
+**Async works too.** `@brooder.record` and `instrument(...)` handle `async def` agents and async
+clients — `AsyncOpenAI`, `AsyncAzureOpenAI`, `AsyncAnthropic`, and Google's `generate_content_async`
+— with no extra setup (the recording context follows your `await`s and into child tasks):
+
+```python
+client = brooder.instrument(openai.AsyncOpenAI())
+
+@brooder.record("support-agent")
+async def agent(question: str) -> str:
+    await client.chat.completions.create(model="gpt-4o", messages=[...])
+    ...
+```
+
+(Async AWS Bedrock via aioboto3 isn't covered yet — the sync boto3 client is.)
+
+## Capture from agent frameworks (OpenTelemetry)
+
+Building on an agent framework? If it emits OpenTelemetry GenAI spans — **LangGraph, CrewAI,
+AutoGen**, and anything else on the convention — add one span processor and Brooder ingests the
+whole trajectory, no manual `tool_call`:
+
+```python
+from opentelemetry import trace
+from brooder.integrations.otel import BrooderSpanProcessor
+
+trace.get_tracer_provider().add_span_processor(BrooderSpanProcessor(agent="support-agent"))
+```
+
+It maps inference spans → turns, `execute_tool` spans → tool calls, and the agent-root span's
+input/output → the case identity and final answer. It also drops straight into the OTel pipelines
+you already run (Datadog / Arize / Honeycomb).
+
+Building directly on the **Claude Agent SDK**? Register Brooder's hooks and it records the tool
+trajectory automatically:
+
+```python
+import brooder
+from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions, ResultMessage
+from brooder.integrations import claude_agent
+
+options = ClaudeAgentOptions(hooks=brooder.claude_agent_hooks(agent="support-agent"))
+async with ClaudeSDKClient(options=options) as client:
+    await client.query(prompt)
+    async for msg in client.receive_response():
+        if isinstance(msg, ResultMessage):
+            claude_agent.record_output(msg.session_id, msg.result)  # optional: capture the answer
+```
+
+`UserPromptSubmit` opens a run (the prompt is the case identity), `PostToolUse` becomes a tool step,
+and `Stop` finalizes it.
+
+On the **OpenAI Agents SDK**? Its tracing is on by default — install Brooder's trace processor once
+and every run is captured (no OpenAI API key required for capture):
+
+```python
+import brooder.integrations.openai_agents as bd_agents
+
+bd_agents.install(agent="support-agent")   # then run your agents as usual
+```
+
+It maps generation/response spans → turns, function spans → tool calls, and handoffs and triggered
+guardrails into the trajectory too — so both tool selection *and* control-flow regressions get
+diffed.
+
+Using **LangChain or LangGraph**? Attach one callback handler — no OpenTelemetry setup required:
+
+```python
+import brooder.integrations.langchain as bd_lc
+
+handler = bd_lc.callback_handler(agent="support-agent")
+graph.invoke({"messages": [...]}, config={"callbacks": [handler]})
+```
+
+The root chain start opens a run (its input is the case identity), model calls become turns, and
+tool calls become tool steps — one handler covers both LangChain and LangGraph.
+
+## It tests agents (the whole trajectory), not single LLM calls
+
+`@brooder.record` wraps your **entire agent** — every step of its plan → act → observe loop.
+The baseline is the full **trajectory**: every tool call across every turn, in order, plus the
+final output. So Brooder catches agent-level regressions, not just token changes in one model
+response.
+
+```bash
+# A multi-step agent that silently stops verifying before answering on the newer model:
+brooder migrate --from gpt-4o --to gpt-5-new examples/loop_agent.py
+# -> REGRESSED: trajectory[1] "verify" removed
+```
+
+That dropped `verify` step happened *inside the loop* — the kind of thing an LLM-output eval
+would never see.
+
+## Why not just use observability / eval tools?
+
+| Tool type | Examples | What it does | The gap Brooder fills |
+| --- | --- | --- | --- |
+| Observability | Langfuse, Laminar, Phoenix | Trace/monitor **after** it runs | Doesn't gate **before** you ship |
+| Eval frameworks | DeepEval, Braintrust, Ragas | Score against **hand-written** datasets | Requires eval authoring nobody maintains |
+| **Brooder** | — | **Record real runs → behavioral diff on every change → CI gate** | **Zero eval-writing, catches model-migration regressions** |
+
+---
+
+## Gate your PRs (GitHub Action)
+
+Drop Brooder into CI and it re-runs your agent on every pull request, comments the behavioral diff,
+and fails the check when behavior regresses. Copy [examples/github-action.yml](examples/github-action.yml)
+to `.github/workflows/brooder.yml`:
+
+```yaml
+permissions:
+  contents: read
+  pull-requests: write        # so it can comment the diff
+
+jobs:
+  agent-snapshot:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: agentbrooder/brooder@v1
+        with:
+          script: tests/agent_snapshot.py
+```
+
+The comment is upserted (updated in place, not spammed) and looks like the `--format markdown`
+output below.
+
+## Machine-readable output (`--json` / OTLP)
+
+`run`, `ci`, and `diff` take `--format table|json|markdown` (`--json` is a shortcut). Exit codes are
+unchanged, so you can gate *and* parse:
+
+```bash
+brooder run agent.py --json | jq '.summary'
+# { "total": 3, "passed": 2, "regressed": 1, "flaky": 0, "regressions": 1, "mean_stability": 80 }
+```
+
+For dashboards, point Brooder at any OTLP endpoint and each run emits a snapshot of gauges
+(`brooder.cases.*`, `brooder.stability.mean`) — **one exporter** that reaches Datadog, Grafana,
+Honeycomb, and CloudWatch:
+
+```bash
+pip install 'brooder[otel]'
+export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318/v1/metrics   # or metrics.otlp_endpoint in brooder.yaml
+brooder ci agent.py
+```
+
+---
+
+## What it checks
+
+- **Structural diff** — the sequence of tool calls, their arguments, and the final output.
+- **Semantic diff** — a pluggable judge (`judge: exact | llm`) so equivalent wording isn't a regression.
+- **Flakiness** — `brooder run --runs 3` runs each case N times and flags non-determinism (`FLAKY`).
+
+Each case gets a verdict — `PASS` / `REGRESSED` / `NEW` / `FLAKY` — and a stability score.
+
+---
+
+## Roadmap
+
+See **[ROADMAP.md](ROADMAP.md)** for what's shipped and what's planned.
+
+## Contributing
+
+See [CONTRIBUTING.md](CONTRIBUTING.md). Issues and PRs welcome — this is being built in public.
+
+## License
+
+[Apache-2.0](LICENSE).

brooder-0.1.0.dist-info/RECORD

@@ -0,0 +1,30 @@
+brooder/__init__.py,sha256=Z2bDpeECu4SryoGTVZFw40mq7MazNGnjYjiNPLQjEe4,946
+brooder/analysis.py,sha256=JR49w0_wRVpDUnMtiARfJ4ll5ADAUl3-QNCind0hcgY,2716
+brooder/cli.py,sha256=xtnN5sXCQHJgx_MoZSaQ_xE7QBgWXDhbh2NVxXh8DiE,9191
+brooder/config.py,sha256=qRfhjH3lZhoryC6wkO5RezLHlTb0A2tS8vZKyJ_6N0k,2911
+brooder/diffing.py,sha256=zoqDrkvJoa6gnSi0aALobz5fYiEdwEwGimS90IX425M,8136
+brooder/errors.py,sha256=GTDorMJiHavHWgDykVKsy4ScCKHa4tyM9ExbPc4UFNE,976
+brooder/judges.py,sha256=MbMcwnQM9viJejPjCEW65XU-iC9ubF5BeYjgP95SJ0A,3696
+brooder/log.py,sha256=JX_0Fd_N3QBjqqGF2WnX9Uh1gApqrusdnZUSdANgNM0,1055
+brooder/metrics.py,sha256=OHDuWUYyTfFU77IhpPYaRE4CaFdZlLR-rzMR3wUz-k0,4398
+brooder/models.py,sha256=IHfYmckuU_cV0rJZgFZloABNvJWrB9A5adFN9e9PS-Q,5131
+brooder/py.typed,sha256=8AkVlIO-WJ6ehRAt6sfsojYu_5YFMRy28xwDjT12VUE,71
+brooder/recorder.py,sha256=V6t8S7kRc93K3SwaYmUKs07_s87IlN_8uyVifiKBneo,13420
+brooder/report.py,sha256=LnIO5gp12r2tyxu4pQD7zwk0NSd9crQbOqdquAVBSSk,9942
+brooder/storage.py,sha256=dqVc5kAafL_Sx9AJLu4-2zSudigqEdfLM0vvXeUpSoo,4641
+brooder/integrations/__init__.py,sha256=syhPKraqalZP-wIwgaStg82GfTL69LazYnSWYpqIL10,2628
+brooder/integrations/anthropic.py,sha256=EvUOPnE9bnE18XcV8_5IKUh3xm_gTABXPl22KXhkBOU,1505
+brooder/integrations/base.py,sha256=Qgqx5M9tUXhMmYiI-e_PAad0vNh9H6e95PKZGX6okrU,6100
+brooder/integrations/bedrock.py,sha256=6S2k1q3G9L5RcKWzsfXtqw5U0YQjgBfF3hV5Yba-ftM,1594
+brooder/integrations/claude_agent.py,sha256=UGqjVZwa_LwVeThaKU9A1GxospvbPftGREViw2dfx4M,6750
+brooder/integrations/google.py,sha256=TQlE5nuzfrRRApz8inZTe9Rrob3yI9_vQRPCuphURzk,2322
+brooder/integrations/langchain.py,sha256=ViB7ORu-cZ2b_ntDUlFc1x-q1FohiIMPZy8lu7Prdr8,12941
+brooder/integrations/openai.py,sha256=YdEhVH30ZW_byT893pT86JcBr0R1KPRDVt6Q66C3Osg,1396
+brooder/integrations/openai_agents.py,sha256=SxjTOeXTDHWx0ZH3FITlYAuG7WpPfKltXoVnqzPxvwE,8032
+brooder/integrations/otel.py,sha256=w-0YjLaHV6fIDLkSwtvCqPlj_EBlxLRhIG_gmFxEc2g,9100
+brooder-0.1.0.dist-info/METADATA,sha256=acAyHgk6xvaX035dgvxuUpLrX_DWwNYUl3oisK8JaaM,13012
+brooder-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+brooder-0.1.0.dist-info/entry_points.txt,sha256=mAUYK2TXw5_8oLgemf05Q7QFjaU02tQmcltHmj-ARZI,45
+brooder-0.1.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+brooder-0.1.0.dist-info/licenses/NOTICE,sha256=FwrMmOBXP-InAZhoIB2VXmxp8m9pVe3c2xiefhnJnX4,295
+brooder-0.1.0.dist-info/RECORD,,

brooder-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

brooder-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+brooder = brooder.cli:main