toolcallcheck 0.1.0__py3-none-any.whl

toolcallcheck/__init__.py

@@ -0,0 +1,71 @@
+"""toolcallcheck: Deterministic, pytest-native testing for tool-using AI agents.
+
+Mock MCP tools, assert exact tool calls and trajectories, verify headers
+and routing, and reproduce failures locally without depending on cloud
+dashboards or live models.
+"""
+
+__version__ = "0.1.0"
+
+# ---------------------------------------------------------------------------
+# P0 Core
+# ---------------------------------------------------------------------------
+from toolcallcheck.assertions import (
+    assert_headers,
+    assert_model_used,
+    assert_no_tool_calls,
+    assert_response_contains,
+    assert_response_equals,
+    assert_response_matches,
+    assert_tool_args_contain,
+    assert_tool_call_count,
+    assert_tool_call_order,
+    assert_tool_calls,
+)
+
+# ---------------------------------------------------------------------------
+# P1 Adoption & Coverage
+# ---------------------------------------------------------------------------
+from toolcallcheck.builders import ScenarioBuilder, ToolResponseBuilder, UserMessageBuilder
+from toolcallcheck.fake_model import FakeModel
+from toolcallcheck.mock_server import MockMCPServer, MockTool
+from toolcallcheck.multi_turn import Conversation
+from toolcallcheck.offline import offline
+from toolcallcheck.plugins import register_assertion, run_custom_assertion
+from toolcallcheck.recording import Recorder
+from toolcallcheck.result import AgentResult, ToolCall
+from toolcallcheck.runner import AgentRunner
+from toolcallcheck.scenario import scenario
+from toolcallcheck.snapshot import assert_snapshot
+from toolcallcheck.trajectory import assert_trajectory
+
+__all__ = [
+    "AgentResult",
+    "AgentRunner",
+    "Conversation",
+    "FakeModel",
+    "MockMCPServer",
+    "MockTool",
+    "Recorder",
+    "ScenarioBuilder",
+    "ToolCall",
+    "ToolResponseBuilder",
+    "UserMessageBuilder",
+    "__version__",
+    "assert_headers",
+    "assert_model_used",
+    "assert_no_tool_calls",
+    "assert_response_contains",
+    "assert_response_equals",
+    "assert_response_matches",
+    "assert_snapshot",
+    "assert_tool_args_contain",
+    "assert_tool_call_count",
+    "assert_tool_call_order",
+    "assert_tool_calls",
+    "assert_trajectory",
+    "offline",
+    "register_assertion",
+    "run_custom_assertion",
+    "scenario",
+]

toolcallcheck/adapters/__init__.py

@@ -0,0 +1,130 @@
+"""Framework adapter protocol and stubs.
+
+Defines the base protocol for adapting toolcallcheck to different
+agent frameworks: OpenAI Agents, LangGraph, PydanticAI, CrewAI.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Protocol, runtime_checkable
+
+from toolcallcheck.result import AgentResult
+
+
+@runtime_checkable
+class FrameworkAdapter(Protocol):
+    """Protocol that framework adapters must implement.
+
+    An adapter bridges between toolcallcheck's mock infrastructure and
+    a specific agent framework's execution model.
+    """
+
+    @property
+    def framework_name(self) -> str:
+        """Return the name of the framework this adapter supports."""
+        ...
+
+    async def invoke(
+        self,
+        message: str,
+        *,
+        tools: list[dict[str, Any]] | None = None,
+        headers: dict[str, str] | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> AgentResult:
+        """Run the agent with the given message and return a result."""
+        ...
+
+
+class OpenAIAgentsAdapter:
+    """Stub adapter for OpenAI Agents SDK.
+
+    This is a placeholder for the full integration. Override
+    ``invoke()`` to wire up the OpenAI Agents SDK.
+    """
+
+    @property
+    def framework_name(self) -> str:
+        return "openai-agents"
+
+    async def invoke(
+        self,
+        message: str,
+        *,
+        tools: list[dict[str, Any]] | None = None,
+        headers: dict[str, str] | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> AgentResult:
+        raise NotImplementedError(
+            "OpenAI Agents adapter is a stub. "
+            "Implement invoke() to connect to the OpenAI Agents SDK."
+        )
+
+
+class LangGraphAdapter:
+    """Stub adapter for LangGraph."""
+
+    @property
+    def framework_name(self) -> str:
+        return "langgraph"
+
+    async def invoke(
+        self,
+        message: str,
+        *,
+        tools: list[dict[str, Any]] | None = None,
+        headers: dict[str, str] | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> AgentResult:
+        raise NotImplementedError(
+            "LangGraph adapter is a stub. Implement invoke() to connect to LangGraph."
+        )
+
+
+class PydanticAIAdapter:
+    """Stub adapter for PydanticAI."""
+
+    @property
+    def framework_name(self) -> str:
+        return "pydantic-ai"
+
+    async def invoke(
+        self,
+        message: str,
+        *,
+        tools: list[dict[str, Any]] | None = None,
+        headers: dict[str, str] | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> AgentResult:
+        raise NotImplementedError(
+            "PydanticAI adapter is a stub. Implement invoke() to connect to PydanticAI."
+        )
+
+
+class CrewAIAdapter:
+    """Stub adapter for CrewAI."""
+
+    @property
+    def framework_name(self) -> str:
+        return "crewai"
+
+    async def invoke(
+        self,
+        message: str,
+        *,
+        tools: list[dict[str, Any]] | None = None,
+        headers: dict[str, str] | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> AgentResult:
+        raise NotImplementedError(
+            "CrewAI adapter is a stub. Implement invoke() to connect to CrewAI."
+        )
+
+
+__all__ = [
+    "CrewAIAdapter",
+    "FrameworkAdapter",
+    "LangGraphAdapter",
+    "OpenAIAgentsAdapter",
+    "PydanticAIAdapter",
+]

toolcallcheck/assertions.py

@@ -0,0 +1,232 @@
+"""Assertion helpers for toolcallcheck.
+
+Every function raises ``AssertionError`` with a structured, human-readable
+diff message on failure so that CI output is immediately actionable.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from toolcallcheck.diff import format_tool_call_diff, format_value_diff
+from toolcallcheck.result import AgentResult
+
+# ---------------------------------------------------------------------------
+# Tool-call assertions
+# ---------------------------------------------------------------------------
+
+
+def assert_tool_calls(
+    result: AgentResult,
+    expected: list[dict[str, Any]],
+    *,
+    strict_order: bool = True,
+) -> None:
+    """Assert that the agent made exactly the expected tool calls.
+
+    Parameters
+    ----------
+    result:
+        The :class:`AgentResult` to inspect.
+    expected:
+        A list of dicts, each with ``"name"`` and ``"args"`` keys.
+    strict_order:
+        If ``True`` (default), the order of tool calls must match.
+    """
+    actual = [{"name": tc.name, "args": tc.args} for tc in result.tool_calls]
+
+    if strict_order:
+        if actual != expected:
+            diff = format_tool_call_diff(expected, actual)
+            raise AssertionError(f"Tool calls do not match (strict order):\n{diff}")
+    else:
+        # Compare as sets — normalize by sorting
+        def _sort_key(d: dict[str, Any]) -> str:
+            return d["name"] + str(sorted(d.get("args", {}).items()))
+
+        if sorted(actual, key=_sort_key) != sorted(expected, key=_sort_key):
+            diff = format_tool_call_diff(expected, actual)
+            raise AssertionError(f"Tool calls do not match (any order):\n{diff}")
+
+
+def assert_tool_call_count(result: AgentResult, expected_count: int) -> None:
+    """Assert the number of tool calls that occurred."""
+    actual = result.tool_call_count
+    if actual != expected_count:
+        raise AssertionError(
+            f"Expected {expected_count} tool call(s), got {actual}.\n"
+            f"  Actual calls: {result.tool_names}"
+        )
+
+
+def assert_no_tool_calls(result: AgentResult) -> None:
+    """Assert that the agent responded without invoking any tool."""
+    if result.tool_calls:
+        names = result.tool_names
+        raise AssertionError(
+            f"Expected no tool calls, but {len(names)} call(s) were made:\n  Tools called: {names}"
+        )
+
+
+def assert_tool_call_order(result: AgentResult, expected_names: list[str]) -> None:
+    """Assert that tools were called in the specified order.
+
+    Only checks the order of the *named* tools; ignores other calls.
+    """
+    actual_names = result.tool_names
+    if actual_names != expected_names:
+        diff = format_value_diff(expected_names, actual_names, label="tool call order")
+        raise AssertionError(f"Tool call order mismatch:\n{diff}")
+
+
+def assert_tool_args_contain(
+    result: AgentResult,
+    tool_name: str,
+    partial_args: dict[str, Any],
+    *,
+    call_index: int = 0,
+) -> None:
+    """Assert that a tool call's arguments contain the given subset.
+
+    Useful when the agent adds auto-generated fields (timestamps, IDs)
+    that you don't want to assert on.
+
+    Parameters
+    ----------
+    result:
+        The :class:`AgentResult` to inspect.
+    tool_name:
+        Name of the tool to check.
+    partial_args:
+        A dict of key-value pairs that must be present in the actual args.
+    call_index:
+        If the tool was called multiple times, which invocation to check
+        (0-indexed, default 0).
+    """
+    matching = result.get_all_tool_calls(tool_name)
+    if not matching:
+        raise AssertionError(
+            f"Tool '{tool_name}' was never called.\n  Actual calls: {result.tool_names}"
+        )
+
+    if call_index >= len(matching):
+        raise AssertionError(
+            f"Tool '{tool_name}' was called {len(matching)} time(s), "
+            f"but call_index={call_index} requested."
+        )
+
+    actual_args = matching[call_index].args
+    missing: dict[str, Any] = {}
+    mismatched: dict[str, tuple[Any, Any]] = {}
+
+    for key, expected_val in partial_args.items():
+        if key not in actual_args:
+            missing[key] = expected_val
+        elif actual_args[key] != expected_val:
+            mismatched[key] = (expected_val, actual_args[key])
+
+    if missing or mismatched:
+        parts: list[str] = [f"Partial argument mismatch for tool '{tool_name}':"]
+        if missing:
+            parts.append(f"  Missing keys: {missing}")
+        if mismatched:
+            for k, (exp, act) in mismatched.items():
+                parts.append(f"  Key '{k}': expected {exp!r}, got {act!r}")
+        raise AssertionError("\n".join(parts))
+
+
+# ---------------------------------------------------------------------------
+# Response assertions
+# ---------------------------------------------------------------------------
+
+
+def assert_response_contains(result: AgentResult, substring: str) -> None:
+    """Assert that the agent's text response contains the given substring."""
+    if substring not in result.response:
+        raise AssertionError(
+            f"Response does not contain expected substring.\n"
+            f"  Expected substring: {substring!r}\n"
+            f"  Actual response:    {result.response!r}"
+        )
+
+
+def assert_response_matches(result: AgentResult, pattern: str) -> None:
+    """Assert that the agent's text response matches the given regex pattern."""
+    if not re.search(pattern, result.response):
+        raise AssertionError(
+            f"Response does not match expected pattern.\n"
+            f"  Pattern:         {pattern!r}\n"
+            f"  Actual response: {result.response!r}"
+        )
+
+
+def assert_response_equals(result: AgentResult, expected: str) -> None:
+    """Assert that the agent's text response exactly equals the expected string."""
+    if result.response != expected:
+        diff = format_value_diff(expected, result.response, label="response")
+        raise AssertionError(f"Response does not match exactly:\n{diff}")
+
+
+# ---------------------------------------------------------------------------
+# Model / routing assertions
+# ---------------------------------------------------------------------------
+
+
+def assert_model_used(result: AgentResult, expected_model: str) -> None:
+    """Assert that the agent used the expected model or routing strategy."""
+    if result.model_used != expected_model:
+        raise AssertionError(f"Expected model '{expected_model}', got '{result.model_used}'.")
+
+
+# ---------------------------------------------------------------------------
+# Header assertions
+# ---------------------------------------------------------------------------
+
+
+def assert_headers(
+    result: AgentResult,
+    expected_headers: dict[str, str],
+    *,
+    exact: bool = False,
+) -> None:
+    """Assert that the captured request headers contain the expected values.
+
+    Parameters
+    ----------
+    result:
+        The :class:`AgentResult` to inspect.
+    expected_headers:
+        Key-value pairs that must be present.
+    exact:
+        If ``True``, the headers must match exactly with no extra keys.
+    """
+    actual = result.headers
+
+    if exact and set(actual.keys()) != set(expected_headers.keys()):
+        extra = set(actual.keys()) - set(expected_headers.keys())
+        missing_keys = set(expected_headers.keys()) - set(actual.keys())
+        parts = ["Headers do not match exactly:"]
+        if extra:
+            parts.append(f"  Unexpected keys: {extra}")
+        if missing_keys:
+            parts.append(f"  Missing keys: {missing_keys}")
+        raise AssertionError("\n".join(parts))
+
+    missing: dict[str, str] = {}
+    mismatched: dict[str, tuple[str, str | None]] = {}
+
+    for key, expected_val in expected_headers.items():
+        if key not in actual:
+            missing[key] = expected_val
+        elif actual[key] != expected_val:
+            mismatched[key] = (expected_val, actual[key])
+
+    if missing or mismatched:
+        parts = ["Header assertion failed:"]
+        if missing:
+            parts.append(f"  Missing headers: {missing}")
+        if mismatched:
+            for k, (exp, act) in mismatched.items():
+                parts.append(f"  Header '{k}': expected {exp!r}, got {act!r}")
+        raise AssertionError("\n".join(parts))

toolcallcheck/builders.py

@@ -0,0 +1,265 @@
+"""Test data builders and factories.
+
+Fluent APIs for constructing test messages, tool responses, and complete
+test scenarios without repetitive boilerplate.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from toolcallcheck.fake_model import FakeModel
+from toolcallcheck.mock_server import MockMCPServer, MockTool
+from toolcallcheck.runner import AgentRunner
+
+
+class UserMessageBuilder:
+    """Fluent builder for constructing agent invocation parameters.
+
+    Usage::
+
+        msg = (
+            UserMessageBuilder("Create user Jane")
+            .with_token("test-jwt")
+            .with_site("1234")
+            .with_header("X-Custom", "value")
+            .build()
+        )
+        result = await runner.invoke(**msg)
+    """
+
+    def __init__(self, message: str) -> None:
+        self._message = message
+        self._token: str | None = None
+        self._site_id: str | None = None
+        self._headers: dict[str, str] = {}
+        self._metadata: dict[str, Any] = {}
+
+    def with_token(self, token: str) -> UserMessageBuilder:
+        """Set the access token."""
+        self._token = token
+        return self
+
+    def with_site(self, site_id: str) -> UserMessageBuilder:
+        """Set the site ID."""
+        self._site_id = site_id
+        return self
+
+    def with_header(self, key: str, value: str) -> UserMessageBuilder:
+        """Add a request header."""
+        self._headers[key] = value
+        return self
+
+    def with_metadata(self, key: str, value: Any) -> UserMessageBuilder:
+        """Add metadata."""
+        self._metadata[key] = value
+        return self
+
+    def build(self) -> dict[str, Any]:
+        """Build the invocation kwargs dict."""
+        result: dict[str, Any] = {"message": self._message}
+        if self._token:
+            result["access_token"] = self._token
+        if self._site_id:
+            result["site_id"] = self._site_id
+        if self._headers:
+            result["headers"] = self._headers
+        if self._metadata:
+            result["metadata"] = self._metadata
+        return result
+
+
+class ToolResponseBuilder:
+    """Fluent builder for constructing mock tool responses.
+
+    Usage::
+
+        tool = (
+            ToolResponseBuilder("create_user")
+            .with_description("Create a new user")
+            .with_param("firstName", "string")
+            .with_param("email", "string")
+            .with_response({"status": "success"})
+            .build()
+        )
+    """
+
+    def __init__(self, name: str) -> None:
+        self._name = name
+        self._description = ""
+        self._params: dict[str, Any] = {}
+        self._response: Any = None
+        self._error: str | dict[str, Any] | None = None
+        self._error_after: int | None = None
+
+    def with_description(self, desc: str) -> ToolResponseBuilder:
+        """Set the tool description."""
+        self._description = desc
+        return self
+
+    def with_param(
+        self,
+        name: str,
+        type_: str = "string",
+        *,
+        required: bool = True,
+    ) -> ToolResponseBuilder:
+        """Add a parameter definition."""
+        self._params[name] = {"type": type_, "required": required}
+        return self
+
+    def with_response(self, response: Any) -> ToolResponseBuilder:
+        """Set the canned success response."""
+        self._response = response
+        return self
+
+    def with_conditional_response(self, fn: Any) -> ToolResponseBuilder:
+        """Set a conditional response function ``(args) -> response``."""
+        self._response = fn
+        return self
+
+    def with_error(
+        self,
+        error: str | dict[str, Any],
+        *,
+        after: int | None = None,
+    ) -> ToolResponseBuilder:
+        """Set error injection."""
+        self._error = error
+        self._error_after = after
+        return self
+
+    def build(self) -> MockTool:
+        """Build the MockTool."""
+        return MockTool(
+            name=self._name,
+            description=self._description,
+            parameters=self._params,
+            response=self._response,
+            error=self._error,
+            error_after=self._error_after,
+        )
+
+
+class ScenarioBuilder:
+    """Fluent builder for constructing complete test scenarios.
+
+    Usage::
+
+        scenario = (
+            ScenarioBuilder("user_flow")
+            .with_tool(
+                ToolResponseBuilder("create_user")
+                .with_response({"status": "ok"})
+                .build()
+            )
+            .with_model_responses([
+                {"tool_calls": [{"name": "create_user", "args": {"email": "a@example.com"}}]},
+                {"content": "Done!"},
+            ])
+            .with_message("Create user a@example.com")
+            .build()
+        )
+        result = await scenario.runner.invoke(**scenario.invocation)
+    """
+
+    def __init__(self, name: str) -> None:
+        self._name = name
+        self._tools: list[MockTool] = []
+        self._model_responses: list[dict[str, Any]] = []
+        self._model_rules: list[tuple[str, dict[str, Any]]] = []
+        self._message: str = ""
+        self._token: str | None = None
+        self._site_id: str | None = None
+        self._headers: dict[str, str] = {}
+        self._model_name: str | None = None
+
+    def with_tool(self, tool: MockTool) -> ScenarioBuilder:
+        """Add a mock tool."""
+        self._tools.append(tool)
+        return self
+
+    def with_model_responses(self, responses: list[dict[str, Any]]) -> ScenarioBuilder:
+        """Set scripted model responses."""
+        self._model_responses = responses
+        return self
+
+    def with_model_rules(self, rules: list[tuple[str, dict[str, Any]]]) -> ScenarioBuilder:
+        """Set pattern-matching model rules."""
+        self._model_rules = rules
+        return self
+
+    def with_message(self, message: str) -> ScenarioBuilder:
+        """Set the user message."""
+        self._message = message
+        return self
+
+    def with_token(self, token: str) -> ScenarioBuilder:
+        """Set the access token."""
+        self._token = token
+        return self
+
+    def with_site_id(self, site_id: str) -> ScenarioBuilder:
+        """Set the site ID."""
+        self._site_id = site_id
+        return self
+
+    def with_header(self, key: str, value: str) -> ScenarioBuilder:
+        """Add a request header."""
+        self._headers[key] = value
+        return self
+
+    def with_model_name(self, name: str) -> ScenarioBuilder:
+        """Set the model name for assertions."""
+        self._model_name = name
+        return self
+
+    def build(self) -> BuiltScenario:
+        """Build the scenario."""
+        server = MockMCPServer()
+        server.add_tools(self._tools)
+
+        model = FakeModel(
+            responses=self._model_responses,
+            rules=self._model_rules,
+        )
+
+        runner = AgentRunner(
+            mcp_server=server,
+            model=model,
+            model_name=self._model_name,
+        )
+
+        invocation: dict[str, Any] = {"message": self._message}
+        if self._token:
+            invocation["access_token"] = self._token
+        if self._site_id:
+            invocation["site_id"] = self._site_id
+        if self._headers:
+            invocation["headers"] = self._headers
+
+        return BuiltScenario(
+            name=self._name,
+            runner=runner,
+            server=server,
+            model=model,
+            invocation=invocation,
+        )
+
+
+class BuiltScenario:
+    """Result of a :class:`ScenarioBuilder.build()` call."""
+
+    def __init__(
+        self,
+        name: str,
+        runner: AgentRunner,
+        server: MockMCPServer,
+        model: FakeModel,
+        invocation: dict[str, Any],
+    ) -> None:
+        self.name = name
+        self.runner = runner
+        self.server = server
+        self.model = model
+        self.invocation = invocation