brooder 0.1.0__py3-none-any.whl

brooder/integrations/otel.py

@@ -0,0 +1,216 @@
+"""OpenTelemetry GenAI ingestion — capture agent trajectories from OTel spans.
+
+Add :class:`BrooderSpanProcessor` to your tracer provider and Brooder turns the GenAI/agent spans
+your stack already emits into a diffable :class:`~brooder.models.Step` trajectory — no manual
+``tool_call``. Because frameworks like **LangGraph, CrewAI and AutoGen** emit OpenTelemetry GenAI
+spans, this one adapter covers all of them, and it slots into the OTel pipelines you already run
+(Datadog / Arize / Honeycomb).
+
+    from opentelemetry import trace
+    from brooder.integrations.otel import BrooderSpanProcessor
+
+    trace.get_tracer_provider().add_span_processor(BrooderSpanProcessor(agent="support-agent"))
+
+The processor is **duck-typed** against the OTel SDK's ``SpanProcessor`` protocol, so importing this
+module does not require ``opentelemetry`` to be installed. Registration is explicit — Brooder never
+mutates your tracer provider for you.
+
+**How spans map to steps** (driven by ``gen_ai.operation.name``):
+
+- inference (``chat`` / ``text_completion`` / ``generate_content``) → ``TURN``
+- ``execute_tool`` → ``TOOL`` (name from ``gen_ai.tool.name``; args/result if present)
+- the agent-root span (``invoke_agent`` or the trace's parentless span) → the run boundary: its
+  input becomes the case identity, its output the ``FINAL`` step.
+
+**Content is opt-in.** Per the GenAI conventions, tool arguments/results and message content are
+only emitted when content capture is enabled (it can be sensitive). Brooder always captures tool
+*names* (the core diff signal) and degrades gracefully when content is absent — but a stable case id
+needs the input, so enable content capture *or* set a ``brooder.case_id`` attribute on the root
+span. A one-time warning is logged when a trace carries no content.
+"""
+
+from __future__ import annotations
+
+import json
+import threading
+from typing import Any, Optional
+
+from .. import recorder
+from ..log import get_logger
+from ..models import Step, StepKind
+from .base import parse_json
+
+_log = get_logger()
+
+# GenAI semantic-convention attribute keys (see the OpenTelemetry GenAI conventions).
+_OP = "gen_ai.operation.name"
+_TOOL_NAME = "gen_ai.tool.name"
+_TOOL_ARGS = "gen_ai.tool.call.arguments"
+_TOOL_RESULT = "gen_ai.tool.call.result"
+_AGENT_NAME = "gen_ai.agent.name"
+_PROVIDER = "gen_ai.provider.name"  # newer key; falls back to gen_ai.system
+_SYSTEM = "gen_ai.system"
+_INPUT_MESSAGES = "gen_ai.input.messages"
+_OUTPUT_MESSAGES = "gen_ai.output.messages"
+_CASE_ID = "brooder.case_id"  # Brooder escape hatch: a caller-set stable case key
+
+_INFERENCE_OPS = frozenset({"chat", "text_completion", "generate_content"})
+
+
+def _attrs(span: Any) -> Any:
+    """Return a span's attribute mapping (empty mapping if it has none)."""
+    return getattr(span, "attributes", None) or {}
+
+
+def _start_time(span: Any) -> int:
+    """Return a span's start time in ns for ordering (0 if unknown)."""
+    return getattr(span, "start_time", 0) or 0
+
+
+def _span_suffix(span: Any, prefix: str) -> Optional[str]:
+    """Return the ``<name>`` part of a ``"<prefix> <name>"`` span name, or None."""
+    name = getattr(span, "name", "") or ""
+    head = prefix + " "
+    return name[len(head) :] if name.startswith(head) else None
+
+
+def _coerce(raw: Any) -> Any:
+    """Decode a JSON-string attribute into structured data; pass other values through."""
+    if isinstance(raw, str):
+        try:
+            return json.loads(raw)
+        except json.JSONDecodeError:
+            return raw
+    return raw
+
+
+def _root_span(spans: list[Any]) -> Any:
+    """Pick the agent-boundary span: an ``invoke_agent`` span, else the parentless one, else 1st."""
+    for span in spans:
+        if _attrs(span).get(_OP) == "invoke_agent":
+            return span
+    for span in spans:
+        if getattr(span, "parent", None) is None:
+            return span
+    return spans[0]
+
+
+def _extract_inputs(root: Any) -> Any:
+    """Extract the run's inputs (for case identity) from the root span, or None if unavailable."""
+    attrs = _attrs(root)
+    if _CASE_ID in attrs:
+        return {"case_id": attrs[_CASE_ID]}
+    raw = attrs.get(_INPUT_MESSAGES)
+    return _coerce(raw) if raw is not None else None
+
+
+def _extract_output(root: Any, spans: list[Any]) -> Any:
+    """Extract the final output from the root span, falling back to the last inference span."""
+    raw = _attrs(root).get(_OUTPUT_MESSAGES)
+    if raw is not None:
+        return _coerce(raw)
+    for span in reversed(spans):
+        attrs = _attrs(span)
+        if attrs.get(_OP) in _INFERENCE_OPS and attrs.get(_OUTPUT_MESSAGES) is not None:
+            return _coerce(attrs[_OUTPUT_MESSAGES])
+    return None
+
+
+def _to_step(span: Any) -> tuple[Optional[Step], bool]:
+    """Map a span to a step (or None to skip); the bool is whether it carried content."""
+    attrs = _attrs(span)
+    op = attrs.get(_OP)
+    if op in _INFERENCE_OPS:
+        provider = attrs.get(_PROVIDER) or attrs.get(_SYSTEM)
+        meta = {"provider": provider} if provider else {}
+        return Step(kind=StepKind.TURN, name="turn", args=meta), False
+    if op == "execute_tool" or _TOOL_NAME in attrs:
+        name = attrs.get(_TOOL_NAME) or _span_suffix(span, "execute_tool") or "tool"
+        raw_args = attrs.get(_TOOL_ARGS)
+        raw_result = attrs.get(_TOOL_RESULT)
+        args = parse_json(raw_args) if raw_args is not None else {}
+        result = _coerce(raw_result) if raw_result is not None else None
+        had_content = raw_args is not None or raw_result is not None
+        return Step(kind=StepKind.TOOL, name=name, args=args, result=result), had_content
+    # invoke_agent / create_agent / non-GenAI spans are the boundary or noise, not a step.
+    return None, False
+
+
+class BrooderSpanProcessor:
+    """An OpenTelemetry ``SpanProcessor`` that ingests GenAI spans into Brooder trajectories.
+
+    Spans are buffered per trace; when the trace's root span ends they are sorted by start time and
+    mapped into one run (see the module docstring for the mapping). Buffering + sort makes
+    out-of-order and cross-thread ``on_end`` delivery a non-issue. Incomplete traces (no parentless
+    span seen) are drained on :meth:`force_flush` / :meth:`shutdown`.
+
+    Args:
+        agent: Logical agent name used to group baselines. If omitted, falls back to the root span's
+            ``gen_ai.agent.name``, then its span name, then ``"otel-agent"``.
+    """
+
+    def __init__(self, agent: Optional[str] = None) -> None:
+        """Create a processor that records into runs named ``agent`` (see the class docstring)."""
+        self._agent = agent
+        self._buffers: dict[int, list[Any]] = {}
+        self._lock = threading.Lock()
+        self._warned_no_content = False
+
+    def on_start(self, span: Any, parent_context: Any = None) -> None:
+        """Part of the ``SpanProcessor`` protocol; capture happens at :meth:`on_end`."""
+        return None
+
+    def on_end(self, span: Any) -> None:
+        """Buffer an ended span under its trace id, finalizing the run when the root span ends."""
+        trace_id = getattr(getattr(span, "context", None), "trace_id", None)
+        if trace_id is None:
+            return
+        with self._lock:
+            self._buffers.setdefault(trace_id, []).append(span)
+            is_root = getattr(span, "parent", None) is None
+        if is_root:
+            self._finalize(trace_id)
+
+    def shutdown(self) -> None:
+        """Drain every buffered (possibly incomplete) trace into a run."""
+        for trace_id in list(self._buffers):
+            self._finalize(trace_id)
+
+    def force_flush(self, timeout_millis: int = 30000) -> bool:
+        """Drain every buffered trace; always returns ``True``."""
+        self.shutdown()
+        return True
+
+    def _finalize(self, trace_id: int) -> None:
+        """Turn a trace's buffered spans into one recorded run."""
+        with self._lock:
+            spans = self._buffers.pop(trace_id, None)
+        if not spans:
+            return
+        spans.sort(key=_start_time)
+
+        root = _root_span(spans)
+        agent = self._agent or _attrs(root).get(_AGENT_NAME) or getattr(root, "name", None)
+        inputs = _extract_inputs(root)
+        handle = recorder.open_run(
+            agent or "otel-agent", inputs=inputs, external_id=f"{trace_id:032x}"
+        )
+
+        saw_content = inputs is not None
+        for span in spans:
+            step, had_content = _to_step(span)
+            if step is not None:
+                handle.add_step(step)
+            saw_content = saw_content or had_content
+
+        output = _extract_output(root, spans)
+        if output is not None:
+            saw_content = True
+        if not saw_content and not self._warned_no_content:
+            self._warned_no_content = True
+            _log.warning(
+                "brooder: OTel spans carry no GenAI content (tool args/results, messages); "
+                "capturing tool names only. Enable content capture, or set a 'brooder.case_id' "
+                "span attribute, for stable and richer baselines."
+            )
+        handle.finish(output)

brooder/judges.py

@@ -0,0 +1,109 @@
+"""Output-equivalence judges.
+
+Structural diffing (tool calls) is always exact. *Output* equivalence, however, is where
+agents are legitimately non-deterministic ("I've started your refund" vs "Your refund is on
+its way" mean the same thing). A :class:`Judge` decides whether two outputs are equivalent.
+
+- ``ExactJudge`` (default): byte-for-byte equality. Deterministic, offline, zero cost.
+- ``LLMJudge``: asks a model whether two outputs are semantically equivalent. Provider-agnostic
+  via an injected ``complete`` callable, so it stays testable and never hard-depends on a vendor.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any, Optional, Protocol
+
+from .log import get_logger
+
+_log = get_logger()
+
+
+class Judge(Protocol):
+    """Decides whether a current output is equivalent to the baseline output."""
+
+    def equivalent(self, baseline: Any, current: Any) -> bool:
+        """Return ``True`` if ``current`` is an acceptable substitute for ``baseline``.
+
+        Args:
+            baseline: The golden output recorded previously.
+            current: The output produced by the run under test.
+
+        Returns:
+            ``True`` if the two outputs are considered equivalent.
+        """
+        ...
+
+
+class ExactJudge:
+    """Equivalent iff equal. The safe, deterministic default."""
+
+    def equivalent(self, baseline: Any, current: Any) -> bool:
+        """Return ``True`` iff the outputs are equal.
+
+        Args:
+            baseline: The golden output.
+            current: The output under test.
+
+        Returns:
+            ``True`` when ``baseline == current``.
+        """
+        return bool(baseline == current)
+
+
+_PROMPT = (
+    "You are grading an AI agent's output against a known-good baseline. "
+    "Answer with exactly 'yes' if they are semantically equivalent for an end user, "
+    "or 'no' if the meaning or outcome differs.\n\n"
+    "BASELINE:\n{baseline}\n\nCURRENT:\n{current}\n\nEquivalent?"
+)
+
+
+class LLMJudge:
+    """Judge semantic equivalence with an LLM via an injected completion function."""
+
+    def __init__(self, complete: Callable[[str], str]) -> None:
+        """Initialize the judge.
+
+        Args:
+            complete: A function that takes a prompt and returns the model's text response.
+                Injecting it keeps the judge provider-agnostic and testable.
+        """
+        self._complete = complete
+
+    def equivalent(self, baseline: Any, current: Any) -> bool:
+        """Return ``True`` if the LLM judges the outputs semantically equivalent.
+
+        Equal outputs short-circuit without calling the backend.
+
+        Args:
+            baseline: The golden output.
+            current: The output under test.
+
+        Returns:
+            ``True`` if outputs are equal, or if the model answers "yes".
+        """
+        if baseline == current:
+            return True
+        prompt = _PROMPT.format(baseline=baseline, current=current)
+        answer = self._complete(prompt).strip().lower()
+        return answer.startswith("y")
+
+
+def make_judge(kind: str, complete: Optional[Callable[[str], str]] = None) -> Judge:
+    """Build a judge from configuration.
+
+    Args:
+        kind: ``"exact"`` or ``"llm"``.
+        complete: The completion backend for ``"llm"``. If omitted, falls back to exact
+            (with a warning) so the tool never hard-fails on a missing backend.
+
+    Returns:
+        A :class:`Judge` implementation.
+    """
+    if kind == "llm":
+        if complete is None:
+            _log.warning("judge: llm is configured but no LLM backend is wired; using exact")
+            return ExactJudge()
+        return LLMJudge(complete)
+    return ExactJudge()

brooder/log.py

@@ -0,0 +1,33 @@
+"""Structured logging for Brooder.
+
+The library logs through the ``brooder`` logger and stays quiet by default (WARNING).
+``brooder -v`` raises it to DEBUG. User-facing output goes through ``report.console``,
+not this logger — logging is for diagnostics.
+"""
+
+from __future__ import annotations
+
+import logging
+
+from rich.logging import RichHandler
+
+_LOGGER_NAME = "brooder"
+_configured = False
+
+
+def setup_logging(verbose: bool = False) -> None:
+    """Configure the ``brooder`` logger once. Idempotent across calls."""
+    global _configured
+    logger = logging.getLogger(_LOGGER_NAME)
+    logger.setLevel(logging.DEBUG if verbose else logging.WARNING)
+    if not _configured:
+        handler = RichHandler(rich_tracebacks=True, show_path=False, show_time=False)
+        handler.setFormatter(logging.Formatter("%(message)s"))
+        logger.addHandler(handler)
+        logger.propagate = False
+        _configured = True
+
+
+def get_logger() -> logging.Logger:
+    """Return the shared ``brooder`` logger."""
+    return logging.getLogger(_LOGGER_NAME)

brooder/metrics.py

@@ -0,0 +1,116 @@
+"""Optional OpenTelemetry (OTLP) metric emission for a Brooder run.
+
+**One OTLP emitter — not per-vendor exporters.** A single ``OTEL_EXPORTER_OTLP_ENDPOINT`` reaches
+Datadog, Grafana, Honeycomb, and CloudWatch (via the ADOT collector), so the OSS core stays lean
+while feeding whatever telemetry backend a team already runs.
+
+Duck-typed like the OTel span processor: the ``opentelemetry`` SDK is imported lazily inside
+:func:`emit`, so the package imports fine without it. If the SDK is missing, :func:`emit` warns once
+and returns — emitting metrics must never fail a CI run.
+
+Each invocation emits a *snapshot* of the run (gauges, not monotonic counters): every ``brooder``
+run reports its own ``total`` / ``regressed`` / ``mean_stability``, which is what a longitudinal
+dashboard wants to chart over time.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Optional
+
+from .log import get_logger
+from .models import Diff
+from .report import diffs_to_summary
+
+_log = get_logger()
+
+# Emit the "SDK not installed" hint at most once per process (this may run in a loop over cases).
+_warned = False
+
+# Default flush budget (ms) so a slow/unreachable collector can't hang CI.
+_FLUSH_TIMEOUT_MS = 5000
+
+
+def _warn_once(message: str) -> None:
+    global _warned
+    if not _warned:
+        _log.warning("brooder: %s", message)
+        _warned = True
+
+
+def _metric_values(diffs: list[Diff]) -> dict[str, int]:
+    """Map diffs to the flat ``{metric_name: value}`` snapshot emitted for a run."""
+    summary = diffs_to_summary(diffs)["summary"]
+    return {
+        "brooder.cases.total": summary["total"],
+        "brooder.cases.passed": summary["passed"],
+        "brooder.cases.regressed": summary["regressed"],
+        "brooder.cases.new": summary["new"],
+        "brooder.cases.flaky": summary["flaky"],
+        "brooder.cases.regressions": summary["regressions"],
+        "brooder.stability.mean": summary["mean_stability"],
+    }
+
+
+def _build_provider(endpoint: str) -> tuple[Any, Any]:
+    """Build an OTLP ``MeterProvider`` and a meter. Raises if the SDK isn't installed.
+
+    The HTTP exporter also reads ``OTEL_EXPORTER_OTLP_ENDPOINT`` itself; we pass ours explicitly.
+    """
+    from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
+    from opentelemetry.sdk.metrics import MeterProvider
+    from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
+
+    reader = PeriodicExportingMetricReader(OTLPMetricExporter(endpoint=endpoint))
+    provider = MeterProvider(metric_readers=[reader])
+    return provider, provider.get_meter("brooder")
+
+
+def _record(meter: Any, values: dict[str, int]) -> None:
+    """Register one observable gauge per metric (collected at ``force_flush``)."""
+    from opentelemetry.metrics import Observation
+
+    def _callback_for(value: int) -> Any:
+        def _observe(_options: Any) -> list[Any]:
+            return [Observation(value)]
+
+        return _observe
+
+    for name, value in values.items():
+        meter.create_observable_gauge(name, callbacks=[_callback_for(value)])
+
+
+def emit(diffs: list[Diff], endpoint: Optional[str] = None) -> None:
+    """Emit a run's result metrics over OTLP, best-effort.
+
+    A no-op unless an endpoint is configured (argument or ``OTEL_EXPORTER_OTLP_ENDPOINT``). If the
+    ``opentelemetry`` SDK isn't installed it warns once and returns; any export error is logged and
+    swallowed. Never raises into the caller — metric emission must not fail a CI run.
+
+    Args:
+        diffs: The per-case diffs to summarize into metrics.
+        endpoint: OTLP endpoint; falls back to ``OTEL_EXPORTER_OTLP_ENDPOINT`` if not given.
+    """
+    endpoint = endpoint or os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT")
+    if not endpoint:
+        return
+
+    try:
+        provider, meter = _build_provider(endpoint)
+    except Exception:
+        _warn_once(
+            "opentelemetry not installed; skipping OTLP metric emission "
+            "(pip install 'brooder[otel]')"
+        )
+        return
+
+    try:
+        _record(meter, _metric_values(diffs))
+        provider.force_flush(_FLUSH_TIMEOUT_MS)
+    except Exception:
+        _log.debug("brooder: OTLP metric emission failed", exc_info=True)
+    finally:
+        try:
+            provider.shutdown()
+        except Exception:
+            _log.debug("brooder: OTLP meter shutdown failed", exc_info=True)

brooder/models.py

@@ -0,0 +1,148 @@
+"""Core data model for Brooder.
+
+A :class:`Run` is one execution of an agent over one input, captured as an ordered
+:class:`Step` **trajectory** (the agent's path through its loop) plus the final output.
+Baselines are golden ``Run``s; a :class:`Diff` compares a fresh ``Run`` to its baseline.
+All models forbid unknown fields so malformed records fail loudly.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from enum import Enum
+from typing import Any, Optional
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+
+class StepKind(str, Enum):
+    """The kind of a step in an agent's trajectory.
+
+    Attributes:
+        TURN: A model turn / decision point.
+        TOOL: A tool invocation (its observation is in ``result``).
+        FINAL: The agent's terminal answer.
+    """
+
+    TURN = "turn"
+    TOOL = "tool"
+    FINAL = "final"
+
+
+class Step(BaseModel):
+    """One step in an agent's trajectory (a model turn, a tool call, or the final answer)."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    kind: StepKind = Field(..., description="Model turn, tool call, or final answer.")
+    name: str = Field(..., min_length=1, description='Tool name, or "turn" / "final".')
+    args: dict[str, Any] = Field(
+        default_factory=dict, description="Tool arguments or turn metadata."
+    )
+    result: Any = Field(default=None, description="Tool observation, or the final output.")
+
+
+class Run(BaseModel):
+    """One execution of an agent over one input, captured as an ordered trajectory."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    agent: str = Field(..., min_length=1, description="Logical name of the agent under test.")
+    case_id: str = Field(..., min_length=1, description="Stable id derived from the inputs.")
+    inputs: Any = Field(default=None, description="The inputs this run was executed with.")
+    model: Optional[str] = Field(default=None, description="Model label (used by `migrate`).")
+    trajectory: list[Step] = Field(
+        default_factory=list, description="Ordered steps the agent took."
+    )
+    output: Any = Field(default=None, description="The agent's final output.")
+    meta: dict[str, Any] = Field(default_factory=dict, description="Free-form metadata.")
+
+    @field_validator("agent")
+    @classmethod
+    def _non_blank_agent(cls, value: str) -> str:
+        value = value.strip()
+        if not value:
+            raise ValueError("agent name must not be blank")
+        return value
+
+    @property
+    def turns(self) -> int:
+        """Number of model turns (``TURN`` steps) in the trajectory."""
+        return sum(1 for step in self.trajectory if step.kind == StepKind.TURN)
+
+    @property
+    def step_count(self) -> int:
+        """Total number of steps in the trajectory."""
+        return len(self.trajectory)
+
+    @property
+    def gave_up(self) -> bool:
+        """True if the run has no ``FINAL`` step — the agent terminated without an answer."""
+        return not any(step.kind == StepKind.FINAL for step in self.trajectory)
+
+    @property
+    def runaway(self) -> bool:
+        """True if the run was aborted for exceeding ``trajectory.max_steps`` (a guardrail hit)."""
+        return bool(self.meta.get("runaway"))
+
+
+class Verdict(str, Enum):
+    """The outcome of comparing a run (or set of runs) to a baseline.
+
+    Attributes:
+        PASS: Behavior matched the baseline.
+        REGRESSED: Behavior changed versus the baseline.
+        NEW: No baseline existed yet for this case.
+        FLAKY: Repeated runs of the same case disagreed with each other.
+    """
+
+    PASS = "pass"
+    REGRESSED = "regressed"
+    NEW = "new"
+    FLAKY = "flaky"
+
+
+class Change(BaseModel):
+    """One difference between a baseline run and the current run."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    path: str = Field(..., description='Where the change is, e.g. "trajectory[1]".')
+    kind: str = Field(..., pattern="^(added|removed|changed)$")
+    before: Any = None
+    after: Any = None
+
+
+class Diff(BaseModel):
+    """The result of comparing a run to its baseline."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    agent: str
+    case_id: str
+    verdict: Verdict
+    changes: list[Change] = Field(default_factory=list)
+    stability: int = Field(default=100, ge=0, le=100, description="100 = identical behavior.")
+
+    @property
+    def ok(self) -> bool:
+        """True if the run matched the baseline (or there was no baseline yet)."""
+        return self.verdict in (Verdict.PASS, Verdict.NEW)
+
+
+def make_case_id(agent: str, inputs: Any) -> str:
+    """Derive a stable case id from an agent name and its inputs.
+
+    The same inputs always map to the same id, so a run can be matched to its baseline
+    across executions.
+
+    Args:
+        agent: The logical agent name.
+        inputs: The inputs the agent was called with (must be JSON-serializable).
+
+    Returns:
+        A 12-character hex digest identifying the case.
+    """
+    payload = json.dumps({"agent": agent, "inputs": inputs}, sort_keys=True, default=str)
+    return hashlib.sha1(payload.encode("utf-8")).hexdigest()[:12]

brooder/py.typed

@@ -0,0 +1 @@
+# Marker file for PEP 561. This package ships inline type information.