brooder 0.1.0__py3-none-any.whl

brooder/recorder.py

@@ -0,0 +1,342 @@
+"""The SDK and the run-lifecycle core.
+
+Public surface: the ``@record`` decorator and ``tool_call`` / ``turn`` capture. The CLI drives
+behavior through env vars set before it executes your script:
+
+    BROODER_MODE       "record" | "run" | "off"
+    BROODER_MODEL      optional label stored on the run (used by `migrate`)
+    BROODER_RUN_INDEX  which repetition this is (used for flakiness detection)
+
+- record : every decorated call is saved as a golden baseline and collected in SESSION.
+- run    : every decorated call is saved as a run and collected in SESSION (diffing happens later).
+- off    : passthrough, no side effects (normal execution of your app).
+
+Each call is captured as an ordered **trajectory** of steps (TURN / TOOL / FINAL). Diffing is
+intentionally *not* done here — see ``analysis`` — so multiple runs of the same case can be grouped
+to detect flakiness.
+
+**Run lifecycle.** A :class:`RunHandle` owns one run's capture: opening it, appending steps,
+finalizing and saving. ``@record`` uses a handle bound to the calling context (so nested
+:func:`tool_call` / :func:`turn` — including provider auto-capture — attach to it). Framework
+adapters instead call :func:`open_run` with the framework's own id (a trace or session), append
+steps as events arrive — possibly from other threads — and :meth:`RunHandle.finish` when the run
+ends; :func:`get_run` looks a handle back up by that id. Both paths feed the same ``Step`` model.
+"""
+
+from __future__ import annotations
+
+import functools
+import inspect
+import os
+import threading
+from collections.abc import Callable
+from contextvars import ContextVar
+from typing import Any, Optional
+
+from . import storage
+from .errors import RunawayError
+from .log import get_logger
+from .models import Run, Step, StepKind, make_case_id
+
+_log = get_logger()
+
+# The handle currently being recorded via @record (so nested steps attach to it).
+_current: ContextVar[Optional[RunHandle]] = ContextVar("brooder_current_run", default=None)
+
+# Runs captured during the most recent CLI invocation.
+SESSION: list[Run] = []
+
+# Adapter-owned runs keyed by an external id (trace_id / session_id), for out-of-band capture.
+_registry: dict[str, RunHandle] = {}
+_registry_lock = threading.Lock()
+
+
+def _mode() -> str:
+    return os.environ.get("BROODER_MODE", "off")
+
+
+def _model() -> Optional[str]:
+    return os.environ.get("BROODER_MODEL")
+
+
+def _max_steps() -> int:
+    """The ``trajectory.max_steps`` guardrail (0 = unlimited), passed in by the CLI."""
+    try:
+        return int(os.environ.get("BROODER_MAX_STEPS", "0"))
+    except ValueError:
+        return 0
+
+
+def reset_session() -> None:
+    """Clear the captured-run buffer and any dangling adapter runs before a new CLI invocation."""
+    SESSION.clear()
+    with _registry_lock:
+        _registry.clear()
+
+
+class RunHandle:
+    """A handle to one run being captured — the single run-lifecycle implementation.
+
+    Append steps with :meth:`turn` / :meth:`tool_call` / :meth:`add_step`, mark the terminal answer
+    with :meth:`final`, and persist with :meth:`finish`. Instances are thread-safe so a framework
+    adapter can receive events on export threads.
+
+    ``@record`` opens a handle with ``raise_on_runaway=True`` so exceeding ``max_steps`` raises
+    :class:`~brooder.errors.RunawayError` (aborting the Python loop). Adapters leave it ``False``:
+    they usually can't cancel the framework's loop, so a runaway is *flagged* (via
+    ``run.meta["runaway"]``) and further steps are dropped, rather than raised into the framework.
+    """
+
+    __slots__ = ("_external_id", "_finished", "_lock", "_raise_on_runaway", "run")
+
+    def __init__(
+        self,
+        run: Run,
+        *,
+        external_id: Optional[str] = None,
+        raise_on_runaway: bool = False,
+    ) -> None:
+        """Wrap ``run`` for capture; see :func:`open_run` for the arguments."""
+        self.run = run
+        self._external_id = external_id
+        self._raise_on_runaway = raise_on_runaway
+        self._finished = False
+        self._lock = threading.Lock()
+
+    def _append(self, step: Step) -> None:
+        """Append a step, enforcing the ``max_steps`` guardrail (raise or flag per the mode)."""
+        with self._lock:
+            cap = _max_steps()
+            if cap and len(self.run.trajectory) >= cap:
+                self.run.meta["runaway"] = True
+                if self._raise_on_runaway:
+                    raise RunawayError(cap)
+                _log.debug(
+                    "run %s/%s hit max_steps=%d; dropping further steps",
+                    self.run.agent,
+                    self.run.case_id,
+                    cap,
+                )
+                return
+            self.run.trajectory.append(step)
+
+    def turn(self, meta: Optional[dict[str, Any]] = None) -> None:
+        """Append a model-turn step (a decision point)."""
+        self._append(Step(kind=StepKind.TURN, name="turn", args=meta or {}))
+
+    def tool_call(
+        self, name: str, args: Optional[dict[str, Any]] = None, result: Any = None
+    ) -> None:
+        """Append a tool-call step; ``result`` is the step's observation."""
+        self._append(Step(kind=StepKind.TOOL, name=name, args=args or {}, result=result))
+
+    def add_step(self, step: Step) -> None:
+        """Append a pre-built :class:`~brooder.models.Step` (escape hatch for adapters)."""
+        self._append(step)
+
+    def set_inputs(self, inputs: Any) -> None:
+        """Set the run inputs after opening and recompute its case id.
+
+        Framework adapters often learn the agent's input mid-run (e.g. from a root span). The case
+        id is derived from the inputs so a run can be matched to its baseline, so it is recomputed.
+        """
+        with self._lock:
+            self.run.inputs = inputs
+            self.run.case_id = make_case_id(self.run.agent, inputs)
+
+    def final(self, output: Any) -> None:
+        """Record the terminal answer: set the run output and append a ``FINAL`` step."""
+        with self._lock:
+            self.run.output = output
+            self.run.trajectory.append(Step(kind=StepKind.FINAL, name="final", result=output))
+
+    def mark_runaway(self) -> None:
+        """Flag the run as a guardrail hit (exceeded ``max_steps``)."""
+        self.run.meta["runaway"] = True
+
+    def finish(self, output: Any = None) -> Run:
+        """Finalize and persist the run. Idempotent — extra calls are no-ops.
+
+        Appends a ``FINAL`` step from ``output`` unless the run is a runaway or gave up (returned
+        no answer). Then saves per the current mode (``record`` → baseline, ``run`` → run) and
+        appends to :data:`SESSION`; ``off`` persists nothing. Unregisters any ``external_id``.
+
+        Args:
+            output: The agent's final output, or ``None`` if it gave up / was aborted.
+
+        Returns:
+            The finalized :class:`~brooder.models.Run`.
+        """
+        with self._lock:
+            if self._finished:
+                return self.run
+            self._finished = True
+
+        if self.run.runaway:
+            _log.debug("run %s/%s aborted: exceeded max_steps", self.run.agent, self.run.case_id)
+        elif output is not None:
+            self.final(output)
+        else:
+            _log.debug(
+                "run %s/%s gave up: finished without a final answer",
+                self.run.agent,
+                self.run.case_id,
+            )
+
+        mode = _mode()
+        if mode == "record":
+            storage.save_baseline(self.run)
+            SESSION.append(self.run)
+            _log.debug("recorded baseline %s/%s", self.run.agent, self.run.case_id)
+        elif mode == "run":
+            storage.save_run(self.run)
+            SESSION.append(self.run)
+            _log.debug("captured run %s/%s", self.run.agent, self.run.case_id)
+        # mode == "off": passthrough, no capture
+
+        if self._external_id is not None:
+            with _registry_lock:
+                _registry.pop(self._external_id, None)
+        return self.run
+
+
+def open_run(
+    agent: str,
+    *,
+    inputs: Any = None,
+    model: Optional[str] = None,
+    external_id: Optional[str] = None,
+    raise_on_runaway: bool = False,
+) -> RunHandle:
+    """Open a run for out-of-band capture from a framework adapter.
+
+    Unlike :func:`record` (which owns a Python call's lifecycle on one thread), this lets an adapter
+    open a run driven by a framework's own events — a trace or a session — and append steps as they
+    arrive, possibly out of order and from other threads. Register it under ``external_id`` and look
+    it back up with :func:`get_run`; call :meth:`RunHandle.finish` when the run ends.
+
+    Args:
+        agent: Logical agent name (groups baselines).
+        inputs: The run inputs; used to derive the case id. If not known yet, set it later with
+            :meth:`RunHandle.set_inputs`.
+        model: Model label; defaults to the CLI-provided ``BROODER_MODEL``.
+        external_id: A stable id (trace/session) to register the handle under for later lookup.
+        raise_on_runaway: If ``True``, exceeding ``max_steps`` raises ``RunawayError`` (used by
+            ``@record``). Adapters leave this ``False`` so a runaway is flagged, not raised into the
+            framework's loop.
+
+    Returns:
+        A :class:`RunHandle`.
+    """
+    run = Run(
+        agent=agent,
+        case_id=make_case_id(agent, inputs),
+        inputs=inputs,
+        model=model or _model(),
+    )
+    handle = RunHandle(run, external_id=external_id, raise_on_runaway=raise_on_runaway)
+    if external_id is not None:
+        with _registry_lock:
+            _registry[external_id] = handle
+    return handle
+
+
+def get_run(external_id: str) -> Optional[RunHandle]:
+    """Return the open run registered under ``external_id``, or ``None`` if there is none."""
+    with _registry_lock:
+        return _registry.get(external_id)
+
+
+def tool_call(name: str, args: Optional[dict[str, Any]] = None, result: Any = None) -> None:
+    """Record a tool call / step into the active recorded run's trajectory.
+
+    Call this from inside a recorded agent for every tool the agent invokes. It is a no-op when
+    called outside a recorded agent, so it is safe to leave in production code.
+
+    Args:
+        name: The tool/function name.
+        args: The arguments passed to the tool.
+        result: The value the tool returned (the step's observation).
+    """
+    handle = _current.get()
+    if handle is not None:
+        handle.tool_call(name, args, result)
+
+
+def turn(meta: Optional[dict[str, Any]] = None) -> None:
+    """Record a model turn (a decision point) into the active recorded run's trajectory.
+
+    Usually emitted automatically by provider auto-capture; call it manually to mark a turn in a
+    hand-instrumented loop. No-op outside a recorded agent.
+
+    Args:
+        meta: Optional metadata to attach to the turn (e.g. the provider).
+    """
+    handle = _current.get()
+    if handle is not None:
+        handle.turn(meta)
+
+
+def _open_recorded(name: str, args: tuple[Any, ...], kwargs: dict[str, Any]) -> RunHandle:
+    """Open a run for a ``@record`` call, deriving its inputs (and case id) from the call args."""
+    return open_run(
+        name, inputs={"args": list(args), "kwargs": dict(kwargs)}, raise_on_runaway=True
+    )
+
+
+def record(name: str) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
+    """Decorate an agent entrypoint so Brooder can record and replay it.
+
+    Works on both plain and ``async def`` agents: an async agent is wrapped with a coroutine so the
+    active-run :class:`~contextvars.ContextVar` stays bound across the agent's ``await`` points (and
+    into any child tasks), letting nested :func:`tool_call` / provider auto-capture attach.
+
+    Args:
+        name: The logical agent name used to group its baselines.
+
+    Returns:
+        A decorator that wraps the agent function; the wrapper captures each call as a
+        :class:`~brooder.models.Run` (its full trajectory) when Brooder is active, and is a
+        passthrough otherwise.
+    """
+
+    def decorator(fn: Callable[..., Any]) -> Callable[..., Any]:
+        if inspect.iscoroutinefunction(fn):
+
+            @functools.wraps(fn)
+            async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
+                handle = _open_recorded(name, args, kwargs)
+                token = _current.set(handle)
+                output = None
+                try:
+                    output = await fn(*args, **kwargs)
+                except RunawayError:
+                    # Guardrail hit: record the run as a runaway; no FINAL answer was produced.
+                    handle.mark_runaway()
+                finally:
+                    _current.reset(token)
+
+                handle.finish(output)
+                return output
+
+            return async_wrapper
+
+        @functools.wraps(fn)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            handle = _open_recorded(name, args, kwargs)
+            token = _current.set(handle)
+            output = None
+            try:
+                output = fn(*args, **kwargs)
+            except RunawayError:
+                # Guardrail hit: record the run as a runaway; no FINAL answer was produced.
+                handle.mark_runaway()
+            finally:
+                _current.reset(token)
+
+            handle.finish(output)
+            return output
+
+        return wrapper
+
+    return decorator

brooder/report.py

@@ -0,0 +1,261 @@
+"""Rendering of diffs — terminal (rich), and machine-readable JSON / Markdown.
+
+The terminal renderers (`print_summary` / `print_diff_detail`) are for humans; `diffs_to_summary` /
+`render_json` / `render_markdown` produce the machine-readable output consumed by CI (`--json`), the
+OTLP metric emitter, and the GitHub Action's PR comment (`--format markdown`).
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Optional
+
+from rich.console import Console
+from rich.panel import Panel
+from rich.table import Table
+
+from .models import Diff, Verdict
+
+console = Console()
+
+# Bump when the shape of `diffs_to_summary` changes, so machine consumers can detect it.
+SUMMARY_SCHEMA_VERSION = 1
+
+_VERDICT: dict[Verdict, tuple[str, str]] = {
+    Verdict.PASS: ("PASS", "green"),
+    Verdict.REGRESSED: ("REGRESSED", "red"),
+    Verdict.NEW: ("NEW", "yellow"),
+    Verdict.FLAKY: ("FLAKY", "magenta"),
+}
+
+
+def _fmt_step(summary: Optional[dict[str, Any]]) -> str:
+    """Render a step summary (``{kind, name, args}``) as e.g. ``TOOL search(q=x)``."""
+    if not summary:
+        return "—"
+    kind = str(summary.get("kind", "")).upper()
+    name = summary.get("name", "")
+    args = summary.get("args") or {}
+    inner = ", ".join(f"{k}={v}" for k, v in list(args.items())[:2])
+    return f"{kind} {name}({inner})" if inner else f"{kind} {name}"
+
+
+def _trajectory_headline(diff: Diff) -> Optional[str]:
+    """The single "path diverged at step N" line, from the first per-step trajectory change."""
+    for change in diff.changes:
+        path = change.path
+        if not (path.startswith("trajectory[") and path.endswith("]")):
+            continue
+        step = path[len("trajectory[") : -1]
+        if change.kind == "removed":
+            return f"path diverged at step {step}: was {_fmt_step(change.before)}, now dropped"
+        if change.kind == "added":
+            return f"path diverged at step {step}: nothing before, now {_fmt_step(change.after)}"
+        return (
+            f"path diverged at step {step}: "
+            f"was {_fmt_step(change.before)}, now {_fmt_step(change.after)}"
+        )
+    return None
+
+
+def _guardrail_line(diff: Diff) -> Optional[str]:
+    """The guardrail terminal note (runaway / gave-up) if the current run newly hit one."""
+    for change in diff.changes:
+        if change.path == "trajectory.runaway" and change.after:
+            return "runaway: the agent exceeded max_steps and was aborted"
+        if change.path == "trajectory.gave_up" and change.after:
+            return "gave up: the agent returned without a final answer"
+    return None
+
+
+def _trajectory_signal(diff: Diff) -> str:
+    """The most salient trajectory signal for the summary table.
+
+    Priority: guardrail terminals (runaway / gave-up) > turn-count > step-count > path change.
+    """
+    for change in diff.changes:
+        if change.path == "trajectory.runaway" and change.after:
+            return "runaway!"
+        if change.path == "trajectory.gave_up" and change.after:
+            return "gave up"
+    for change in diff.changes:
+        if change.path == "trajectory.turns":
+            return f"turns {change.before}→{change.after}"
+    for change in diff.changes:
+        if change.path == "trajectory.steps":
+            return f"steps {change.before}→{change.after}"
+    if any(c.path.startswith("trajectory[") for c in diff.changes):
+        return "path changed"
+    return "—"
+
+
+def print_summary(diffs: list[Diff]) -> None:
+    """Print a one-row-per-case results table.
+
+    Args:
+        diffs: The per-case diffs to summarize.
+    """
+    table = Table(title="Brooder results", title_style="bold")
+    table.add_column("Agent")
+    table.add_column("Case")
+    table.add_column("Verdict")
+    table.add_column("Changes", justify="right")
+    table.add_column("Trajectory")
+    table.add_column("Stability", justify="right")
+    for diff in diffs:
+        label, style = _VERDICT[diff.verdict]
+        table.add_row(
+            diff.agent,
+            diff.case_id,
+            f"[{style}]{label}[/]",
+            str(len(diff.changes)),
+            _trajectory_signal(diff),
+            str(diff.stability),
+        )
+    console.print(table)
+
+
+def print_diff_detail(diff: Diff) -> None:
+    """Print a detailed, color-coded panel of the changes in a single diff.
+
+    Args:
+        diff: The diff to render.
+    """
+    label, style = _VERDICT[diff.verdict]
+    title = f"{diff.agent} · {diff.case_id} · [{style}]{label}[/]"
+    if not diff.changes:
+        console.print(Panel("No behavioral changes.", title=title, border_style=style))
+        return
+
+    lines: list[str] = []
+    guardrail = _guardrail_line(diff)
+    headline = _trajectory_headline(diff)
+    if guardrail:
+        lines.append(f"[bold red]⚠ {guardrail}[/]")
+    if headline:
+        lines.append(f"[bold]{headline}[/]")
+    if lines:
+        lines.append("")
+    for change in diff.changes:
+        if change.kind == "removed":
+            lines.append(f"[red]- {change.path}[/]  {change.before}")
+        elif change.kind == "added":
+            lines.append(f"[green]+ {change.path}[/]  {change.after}")
+        else:
+            lines.append(f"[yellow]~ {change.path}[/]")
+            lines.append(f"    before: {change.before}")
+            lines.append(f"    after:  {change.after}")
+    console.print(Panel("\n".join(lines), title=title, border_style=style))
+
+
+# --- machine-readable output ------------------------------------------------
+
+
+def diffs_to_summary(diffs: list[Diff]) -> dict[str, Any]:
+    """Serialize per-case diffs into a plain, JSON-safe summary dict.
+
+    The single source of truth for machine-readable output: `render_json`, the OTLP metric emitter,
+    and the GitHub Action's Markdown comment all derive from this. Everything comes from the
+    :class:`~brooder.models.Diff` objects alone (verdict, stability, changes); cost/latency are not
+    included because the run model does not capture them yet.
+
+    Args:
+        diffs: The per-case diffs (one per case), as returned by ``analysis.analyze``.
+
+    Returns:
+        ``{schema_version, summary: {...counts, mean_stability}, cases: [...]}``.
+    """
+    counts = {verdict.value: 0 for verdict in Verdict}
+    for diff in diffs:
+        counts[diff.verdict.value] += 1
+    total = len(diffs)
+    mean_stability = round(sum(d.stability for d in diffs) / total) if total else 100
+    # "regressions" is what gates CI: anything that is not PASS/NEW (regressed + flaky).
+    regressions = sum(1 for d in diffs if not d.ok)
+
+    cases = [
+        {
+            "agent": diff.agent,
+            "case_id": diff.case_id,
+            "verdict": diff.verdict.value,
+            "ok": diff.ok,
+            "stability": diff.stability,
+            "change_count": len(diff.changes),
+            "changes": [
+                {"path": c.path, "kind": c.kind, "before": c.before, "after": c.after}
+                for c in diff.changes
+            ],
+        }
+        for diff in diffs
+    ]
+    return {
+        "schema_version": SUMMARY_SCHEMA_VERSION,
+        "summary": {
+            "total": total,
+            "passed": counts[Verdict.PASS.value],
+            "regressed": counts[Verdict.REGRESSED.value],
+            "new": counts[Verdict.NEW.value],
+            "flaky": counts[Verdict.FLAKY.value],
+            "regressions": regressions,
+            "mean_stability": mean_stability,
+        },
+        "cases": cases,
+    }
+
+
+def render_json(diffs: list[Diff]) -> str:
+    """Render diffs as a pretty-printed JSON document (see :func:`diffs_to_summary`)."""
+    return json.dumps(diffs_to_summary(diffs), indent=2, sort_keys=True, default=str)
+
+
+def render_markdown(diffs: list[Diff]) -> str:
+    """Render diffs as a Markdown report suitable for a PR comment.
+
+    A headline + one-row-per-case table, plus a collapsible details block for the regressions.
+    Reuses the same trajectory-signal helpers as the terminal renderer so the two stay consistent.
+    """
+    summary = diffs_to_summary(diffs)["summary"]
+    headline = (
+        "✅ no behavioral regressions"
+        if summary["regressions"] == 0
+        else f"❌ {summary['regressions']} behavioral regression(s)"
+    )
+    lines = [
+        "## Brooder results",
+        "",
+        f"**{headline}** — {summary['total']} case(s): "
+        f"{summary['passed']} passed · {summary['regressed']} regressed · "
+        f"{summary['flaky']} flaky · {summary['new']} new · "
+        f"mean stability {summary['mean_stability']}",
+        "",
+        "| Agent | Case | Verdict | Changes | Trajectory | Stability |",
+        "| --- | --- | --- | ---: | --- | ---: |",
+    ]
+    for diff in diffs:
+        label = _VERDICT[diff.verdict][0]
+        lines.append(
+            f"| {diff.agent} | `{diff.case_id}` | {label} | {len(diff.changes)} "
+            f"| {_trajectory_signal(diff)} | {diff.stability} |"
+        )
+
+    regressions = [d for d in diffs if not d.ok]
+    if regressions:
+        lines += ["", "<details><summary>What changed</summary>", ""]
+        for diff in regressions:
+            lines.append(f"#### {diff.agent} · {diff.case_id} — {_VERDICT[diff.verdict][0]}")
+            guardrail = _guardrail_line(diff)
+            trajectory = _trajectory_headline(diff)
+            if guardrail:
+                lines.append(f"- ⚠ {guardrail}")
+            if trajectory:
+                lines.append(f"- {trajectory}")
+            for change in diff.changes:
+                if change.kind == "removed":
+                    lines.append(f"- `- {change.path}` — {change.before}")
+                elif change.kind == "added":
+                    lines.append(f"- `+ {change.path}` — {change.after}")
+                else:
+                    lines.append(f"- `~ {change.path}` — {change.before} → {change.after}")
+            lines.append("")
+        lines.append("</details>")
+    return "\n".join(lines).rstrip() + "\n"