pytest-agentcontract 0.1.0__py3-none-any.whl

agentcontract/__init__.py

@@ -0,0 +1,25 @@
+"""pytest-agentcontract: Deterministic CI tests for LLM agent trajectories."""
+
+__version__ = "0.1.0"
+
+# Lazy imports to avoid circular dependencies and speed up pytest plugin loading.
+# The plugin.py entry point imports specific modules directly.
+
+
+def __getattr__(name: str):  # noqa: ANN001
+    """Lazy-load public API classes on first access."""
+    _lazy = {
+        "Recorder": "agentcontract.recorder.core",
+        "ReplayEngine": "agentcontract.replay.engine",
+        "AssertionEngine": "agentcontract.assertions.engine",
+        "AgentContractConfig": "agentcontract.config",
+    }
+    if name in _lazy:
+        import importlib
+
+        mod = importlib.import_module(_lazy[name])
+        return getattr(mod, name)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+__all__ = ["Recorder", "ReplayEngine", "AssertionEngine", "AgentContractConfig"]

agentcontract/adapters/__init__.py

@@ -0,0 +1,36 @@
+"""Framework adapters for popular agent libraries.
+
+Each adapter wraps a framework's execution method to record trajectories
+into the agentcontract format. All adapters follow the same pattern:
+
+    unpatch = record_<thing>(target, recorder)
+    # ... run your agent ...
+    unpatch()
+
+Available adapters:
+    - langgraph: LangGraph CompiledGraph (invoke/ainvoke)
+    - llamaindex: LlamaIndex AgentRunner (chat/query)
+    - openai_agents: OpenAI Agents SDK Runner (run/run_sync)
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def __getattr__(name: str) -> Any:
+    """Lazy-load adapters to avoid importing heavy framework dependencies."""
+    _lazy = {
+        "record_graph": "agentcontract.adapters.langgraph",
+        "record_agent": "agentcontract.adapters.llamaindex",
+        "record_runner": "agentcontract.adapters.openai_agents",
+    }
+    if name in _lazy:
+        import importlib
+
+        mod = importlib.import_module(_lazy[name])
+        return getattr(mod, name)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+__all__ = ["record_graph", "record_agent", "record_runner"]

agentcontract/adapters/langgraph.py

@@ -0,0 +1,157 @@
+"""LangGraph adapter for pytest-agentcontract.
+
+Records agent trajectories from LangGraph graph executions by wrapping
+the graph's stream/invoke methods.
+
+Usage:
+    from agentcontract.adapters.langgraph import record_graph
+
+    recorder = Recorder(scenario="customer-support")
+    with recorder.recording():
+        unpatch = record_graph(graph, recorder)
+        result = graph.invoke({"messages": [("user", "I need a refund")]})
+        unpatch()
+        recorder.save("tests/scenarios/customer-support.agentrun.json")
+"""
+
+from __future__ import annotations
+
+import functools
+import time
+from collections.abc import Callable
+from typing import Any
+
+from agentcontract.recorder.core import Recorder
+
+
+def record_graph(graph: Any, recorder: Recorder) -> Callable[[], None]:
+    """Wrap a LangGraph CompiledGraph to record trajectories.
+
+    Intercepts ``invoke()`` and ``ainvoke()`` to capture agent turns,
+    tool calls, and message flow as an AgentRun.
+
+    Returns an unpatch function that restores original methods.
+
+    Args:
+        graph: A LangGraph CompiledGraph (from ``graph.compile()``).
+        recorder: A Recorder instance to capture the trajectory.
+    """
+    original_invoke = getattr(graph, "invoke", None)
+    original_ainvoke = getattr(graph, "ainvoke", None)
+
+    if original_invoke is not None:
+
+        @functools.wraps(original_invoke)
+        def recording_invoke(*args: Any, **kwargs: Any) -> Any:
+            start = time.monotonic()
+            result = original_invoke(*args, **kwargs)
+            latency_ms = (time.monotonic() - start) * 1000
+            _extract_turns(result, recorder, latency_ms)
+            return result
+
+        graph.invoke = recording_invoke
+
+    if original_ainvoke is not None:
+
+        @functools.wraps(original_ainvoke)
+        async def recording_ainvoke(*args: Any, **kwargs: Any) -> Any:
+            start = time.monotonic()
+            result = await original_ainvoke(*args, **kwargs)
+            latency_ms = (time.monotonic() - start) * 1000
+            _extract_turns(result, recorder, latency_ms)
+            return result
+
+        graph.ainvoke = recording_ainvoke
+
+    def unpatch() -> None:
+        if original_invoke is not None:
+            graph.invoke = original_invoke
+        if original_ainvoke is not None:
+            graph.ainvoke = original_ainvoke
+
+    return unpatch
+
+
+def _extract_turns(result: Any, recorder: Recorder, latency_ms: float) -> None:
+    """Extract turns from a LangGraph result dict.
+
+    LangGraph returns a state dict. The standard ``messages`` key contains
+    the conversation as a list of LangChain message objects.
+    """
+    if not isinstance(result, dict):
+        return
+
+    messages = result.get("messages", [])
+    if not isinstance(messages, (list, tuple)):
+        return
+
+    for msg in messages:
+        role = _get_role(msg)
+        content = _get_content(msg)
+        tool_calls = _get_tool_calls(msg)
+
+        if role in ("user", "assistant", "system", "tool"):
+            recorder.add_turn(
+                role=role,
+                content=content,
+                tool_calls=tool_calls or None,
+                latency_ms=latency_ms if role == "assistant" else None,
+            )
+
+
+def _get_role(msg: Any) -> str:
+    """Extract role from a LangChain message object or dict."""
+    if isinstance(msg, dict):
+        return str(msg.get("role", msg.get("type", "")))
+
+    # LangChain message classes: HumanMessage, AIMessage, SystemMessage, ToolMessage
+    type_attr = getattr(msg, "type", None)
+    if type_attr:
+        role_map = {"human": "user", "ai": "assistant", "system": "system", "tool": "tool"}
+        return role_map.get(str(type_attr), str(type_attr))
+
+    return ""
+
+
+def _get_content(msg: Any) -> str | None:
+    """Extract text content from a message."""
+    content = msg.get("content") if isinstance(msg, dict) else getattr(msg, "content", None)
+
+    if content is None:
+        return None
+    if isinstance(content, str):
+        return content or None
+    # LangChain can return list of content blocks
+    if isinstance(content, list):
+        texts = [str(b.get("text", "")) if isinstance(b, dict) else str(b) for b in content]
+        joined = "".join(texts)
+        return joined or None
+    return str(content) or None
+
+
+def _get_tool_calls(msg: Any) -> list[dict[str, Any]] | None:
+    """Extract tool calls from a LangChain AIMessage."""
+    raw = msg.get("tool_calls", []) if isinstance(msg, dict) else getattr(msg, "tool_calls", None)
+
+    if not raw or not isinstance(raw, (list, tuple)):
+        return None
+
+    calls: list[dict[str, Any]] = []
+    for tc in raw:
+        if isinstance(tc, dict):
+            name = tc.get("name", "")
+            args = tc.get("args", tc.get("arguments", {}))
+            call_id = tc.get("id", "")
+        else:
+            name = getattr(tc, "name", "")
+            args = getattr(tc, "args", getattr(tc, "arguments", {}))
+            call_id = getattr(tc, "id", "")
+
+        if name:
+            calls.append({
+                "id": str(call_id) if call_id else "",
+                "function": str(name),
+                "arguments": args if isinstance(args, dict) else {},
+            })
+
+    return calls or None

agentcontract/adapters/llamaindex.py

@@ -0,0 +1,167 @@
+"""LlamaIndex adapter for pytest-agentcontract.
+
+Records agent trajectories from LlamaIndex AgentRunner / ReActAgent
+by wrapping the chat/query methods.
+
+Usage:
+    from agentcontract.adapters.llamaindex import record_agent
+
+    recorder = Recorder(scenario="rag-qa")
+    with recorder.recording():
+        unpatch = record_agent(agent, recorder)
+        response = agent.chat("What's the refund policy?")
+        unpatch()
+        recorder.save("tests/scenarios/rag-qa.agentrun.json")
+"""
+
+from __future__ import annotations
+
+import functools
+import time
+from collections.abc import Callable
+from typing import Any
+
+from agentcontract.recorder.core import Recorder
+
+
+def record_agent(agent: Any, recorder: Recorder) -> Callable[[], None]:
+    """Wrap a LlamaIndex agent to record trajectories.
+
+    Intercepts ``chat()``, ``achat()``, ``query()``, and ``aquery()``
+    to capture the full interaction as an AgentRun.
+
+    Returns an unpatch function that restores original methods.
+
+    Args:
+        agent: A LlamaIndex AgentRunner, ReActAgent, or similar.
+        recorder: A Recorder instance to capture the trajectory.
+    """
+    originals: dict[str, Any] = {}
+    methods = ["chat", "achat", "query", "aquery"]
+
+    for method_name in methods:
+        original = getattr(agent, method_name, None)
+        if original is None:
+            continue
+        originals[method_name] = original
+
+        if method_name.startswith("a"):
+            # Async variant
+            @functools.wraps(original)
+            async def async_wrapper(
+                *args: Any,
+                _orig: Any = original,
+                _name: str = method_name,
+                **kwargs: Any,
+            ) -> Any:
+                start = time.monotonic()
+                result = await _orig(*args, **kwargs)
+                latency_ms = (time.monotonic() - start) * 1000
+                _extract_from_response(result, recorder, latency_ms, agent)
+                return result
+
+            setattr(agent, method_name, async_wrapper)
+        else:
+            # Sync variant
+            @functools.wraps(original)
+            def sync_wrapper(
+                *args: Any,
+                _orig: Any = original,
+                _name: str = method_name,
+                **kwargs: Any,
+            ) -> Any:
+                start = time.monotonic()
+                result = _orig(*args, **kwargs)
+                latency_ms = (time.monotonic() - start) * 1000
+                _extract_from_response(result, recorder, latency_ms, agent)
+                return result
+
+            setattr(agent, method_name, sync_wrapper)
+
+    def unpatch() -> None:
+        for name, orig in originals.items():
+            setattr(agent, name, orig)
+
+    return unpatch
+
+
+def _extract_from_response(
+    response: Any, recorder: Recorder, latency_ms: float, agent: Any
+) -> None:
+    """Extract turns from a LlamaIndex response.
+
+    LlamaIndex agents return AgentChatResponse or Response objects.
+    We also check the agent's chat_history/memory for the full trajectory.
+    """
+    # Try to get the response text
+    response_text = None
+    if hasattr(response, "response"):
+        response_text = str(response.response) if response.response else None
+    elif hasattr(response, "message"):
+        msg = response.message
+        response_text = _get_content(msg)
+
+    # Extract tool calls from sources/source_nodes (tool outputs)
+    tool_calls = _extract_tool_calls(response)
+
+    if response_text or tool_calls:
+        recorder.add_turn(
+            role="assistant",
+            content=response_text,
+            tool_calls=tool_calls or None,
+            latency_ms=latency_ms,
+        )
+
+
+def _extract_tool_calls(response: Any) -> list[dict[str, Any]] | None:
+    """Extract tool calls from LlamaIndex response sources."""
+    calls: list[dict[str, Any]] = []
+
+    # AgentChatResponse has .sources which are ToolOutput objects
+    sources = getattr(response, "sources", None)
+    if sources and isinstance(sources, (list, tuple)):
+        for source in sources:
+            tool_name = getattr(source, "tool_name", None)
+            if not tool_name:
+                continue
+            raw_input = getattr(source, "raw_input", {})
+            raw_output = getattr(source, "raw_output", None)
+
+            calls.append({
+                "id": "",
+                "function": str(tool_name),
+                "arguments": raw_input if isinstance(raw_input, dict) else {},
+                "result": str(raw_output) if raw_output is not None else None,
+            })
+
+    # Also check source_nodes for retrieval-based responses
+    source_nodes = getattr(response, "source_nodes", None)
+    if source_nodes and isinstance(source_nodes, (list, tuple)):
+        for node in source_nodes:
+            node_id = getattr(node, "node_id", "") or getattr(node, "id_", "")
+            score = getattr(node, "score", None)
+            text = ""
+            inner = getattr(node, "node", None) or getattr(node, "text", None)
+            if inner:
+                text = getattr(inner, "text", str(inner))[:200]
+
+            calls.append({
+                "id": str(node_id) if node_id else "",
+                "function": "_retrieve",
+                "arguments": {"score": score} if score is not None else {},
+                "result": text or None,
+            })
+
+    return calls or None
+
+
+def _get_content(msg: Any) -> str | None:
+    """Extract text from a LlamaIndex ChatMessage."""
+    if msg is None:
+        return None
+    content = getattr(msg, "content", None)
+    if content is None:
+        return None
+    if isinstance(content, str):
+        return content or None
+    return str(content) or None

agentcontract/adapters/openai_agents.py

@@ -0,0 +1,260 @@
+"""OpenAI Agents SDK adapter for pytest-agentcontract.
+
+Records agent trajectories from the OpenAI Agents SDK (openai-agents)
+by wrapping Runner.run / Runner.run_sync.
+
+Usage:
+    from agentcontract.adapters.openai_agents import record_runner
+
+    recorder = Recorder(scenario="triage-agent")
+    with recorder.recording():
+        unpatch = record_runner(recorder)
+        result = Runner.run_sync(agent, "I need help with billing")
+        unpatch()
+        recorder.save("tests/scenarios/triage-agent.agentrun.json")
+"""
+
+from __future__ import annotations
+
+import functools
+import time
+from collections.abc import Callable
+from typing import Any
+
+from agentcontract.recorder.core import Recorder
+
+
+def record_runner(recorder: Recorder) -> Callable[[], None]:
+    """Patch the OpenAI Agents SDK Runner class to record trajectories.
+
+    Intercepts ``Runner.run()`` and ``Runner.run_sync()`` at the class level
+    to capture agent runs as AgentRun trajectories.
+
+    Returns an unpatch function that restores original methods.
+
+    Args:
+        recorder: A Recorder instance to capture the trajectory.
+    """
+    try:
+        from agents import Runner  # type: ignore[import-untyped]
+    except ImportError as e:
+        raise ImportError(
+            "OpenAI Agents SDK not installed. Install with: pip install openai-agents"
+        ) from e
+
+    originals: dict[str, Any] = {}
+
+    # Patch run (async)
+    original_run = getattr(Runner, "run", None)
+    if original_run is not None:
+        originals["run"] = original_run
+
+        @functools.wraps(original_run)
+        async def recording_run(*args: Any, **kwargs: Any) -> Any:
+            start = time.monotonic()
+            result = await original_run(*args, **kwargs)
+            latency_ms = (time.monotonic() - start) * 1000
+            _extract_from_result(result, recorder, latency_ms)
+            return result
+
+        Runner.run = recording_run  # type: ignore[assignment]
+
+    # Patch run_sync
+    original_run_sync = getattr(Runner, "run_sync", None)
+    if original_run_sync is not None:
+        originals["run_sync"] = original_run_sync
+
+        @functools.wraps(original_run_sync)
+        def recording_run_sync(*args: Any, **kwargs: Any) -> Any:
+            start = time.monotonic()
+            result = original_run_sync(*args, **kwargs)
+            latency_ms = (time.monotonic() - start) * 1000
+            _extract_from_result(result, recorder, latency_ms)
+            return result
+
+        Runner.run_sync = recording_run_sync  # type: ignore[assignment]
+
+    def unpatch() -> None:
+        for name, orig in originals.items():
+            setattr(Runner, name, orig)
+
+    return unpatch
+
+
+def _extract_from_result(result: Any, recorder: Recorder, latency_ms: float) -> None:
+    """Extract turns from a RunResult.
+
+    The OpenAI Agents SDK RunResult contains:
+    - result.final_output: the agent's final response
+    - result.new_items: list of RunItem objects (messages, tool calls, handoffs)
+    - result.raw_responses: list of ModelResponse objects
+    """
+    if result is None:
+        return
+
+    # Extract from new_items (preferred -- gives full trajectory)
+    new_items = getattr(result, "new_items", None)
+    if new_items and isinstance(new_items, (list, tuple)):
+        _extract_from_items(new_items, recorder, latency_ms)
+        return
+
+    # Fallback: just record the final output
+    final = getattr(result, "final_output", None)
+    if final is not None:
+        content = str(final) if not isinstance(final, str) else final
+        recorder.add_turn(
+            role="assistant",
+            content=content,
+            latency_ms=latency_ms,
+        )
+
+
+def _extract_from_items(
+    items: list[Any] | tuple[Any, ...], recorder: Recorder, latency_ms: float
+) -> None:
+    """Extract turns from RunItem list."""
+    for item in items:
+        item_type = type(item).__name__
+
+        if item_type == "MessageOutputItem":
+            # Agent message output
+            agent_msg = getattr(item, "raw_item", None)
+            content = _extract_message_content(agent_msg)
+            tool_calls = _extract_message_tool_calls(agent_msg)
+            if content or tool_calls:
+                recorder.add_turn(
+                    role="assistant",
+                    content=content,
+                    tool_calls=tool_calls or None,
+                    latency_ms=latency_ms,
+                )
+
+        elif item_type == "ToolCallItem":
+            # Individual tool call
+            raw = getattr(item, "raw_item", None)
+            if raw is not None:
+                name = getattr(raw, "name", "") or _get_nested(raw, "function", "name", default="")
+                args = _get_tool_arguments(raw)
+                if name:
+                    recorder.add_turn(
+                        role="assistant",
+                        tool_calls=[{
+                            "id": str(getattr(raw, "id", "") or getattr(raw, "call_id", "") or ""),
+                            "function": str(name),
+                            "arguments": args,
+                        }],
+                    )
+
+        elif item_type == "ToolCallOutputItem":
+            # Tool result
+            output = getattr(item, "output", None)
+            recorder.add_turn(
+                role="tool",
+                content=str(output) if output is not None else None,
+            )
+
+        elif item_type == "HandoffCallItem":
+            # Agent handoff
+            target = getattr(item, "target_agent", None)
+            target_name = getattr(target, "name", str(target)) if target else "unknown"
+            recorder.add_turn(
+                role="assistant",
+                content=f"[handoff to {target_name}]",
+            )
+
+
+def _extract_message_content(msg: Any) -> str | None:
+    """Extract text content from an Agents SDK message."""
+    if msg is None:
+        return None
+
+    # Check for content list (OpenAI format)
+    content = getattr(msg, "content", None)
+    if content is None:
+        return None
+
+    if isinstance(content, str):
+        return content or None
+
+    if isinstance(content, list):
+        texts = []
+        for block in content:
+            if isinstance(block, dict):
+                if block.get("type") in ("output_text", "text"):
+                    texts.append(str(block.get("text", "")))
+            else:
+                block_type = getattr(block, "type", "")
+                if block_type in ("output_text", "text"):
+                    texts.append(str(getattr(block, "text", "")))
+        joined = "".join(texts)
+        return joined or None
+
+    return str(content) or None
+
+
+def _extract_message_tool_calls(msg: Any) -> list[dict[str, Any]] | None:
+    """Extract tool calls from an Agents SDK message."""
+    if msg is None:
+        return None
+
+    raw_calls = getattr(msg, "tool_calls", None)
+    if not raw_calls or not isinstance(raw_calls, (list, tuple)):
+        return None
+
+    calls: list[dict[str, Any]] = []
+    for tc in raw_calls:
+        name = getattr(tc, "name", "") or _get_nested(tc, "function", "name", default="")
+        if name:
+            calls.append({
+                "id": str(getattr(tc, "id", "") or getattr(tc, "call_id", "") or ""),
+                "function": str(name),
+                "arguments": _get_tool_arguments(tc),
+            })
+
+    return calls or None
+
+
+def _get_tool_arguments(tc: Any) -> dict[str, Any]:
+    """Extract arguments dict from a tool call object."""
+    # Try direct args/arguments
+    for attr in ("args", "arguments", "input"):
+        val = getattr(tc, attr, None)
+        if isinstance(val, dict):
+            return val
+        if isinstance(val, str) and val:
+            import json
+
+            try:
+                parsed = json.loads(val)
+                if isinstance(parsed, dict):
+                    return parsed
+            except (json.JSONDecodeError, TypeError):
+                return {"_raw": val}
+
+    # Try function.arguments (OpenAI chat format)
+    fn = getattr(tc, "function", None)
+    if fn:
+        fn_args = getattr(fn, "arguments", None)
+        if isinstance(fn_args, dict):
+            return fn_args
+        if isinstance(fn_args, str) and fn_args:
+            import json
+
+            try:
+                parsed = json.loads(fn_args)
+                if isinstance(parsed, dict):
+                    return parsed
+            except (json.JSONDecodeError, TypeError):
+                pass
+
+    return {}
+
+
+def _get_nested(obj: Any, *attrs: str, default: Any = None) -> Any:
+    """Safely traverse nested attributes."""
+    current = obj
+    for attr in attrs:
+        if current is None:
+            return default
+        current = getattr(current, attr, None)
+    return current if current is not None else default

agentcontract/assertions/__init__.py

@@ -0,0 +1,5 @@
+"""Assertion engine for validating agent trajectories."""
+
+from agentcontract.assertions.engine import AssertionEngine
+
+__all__ = ["AssertionEngine"]