agentverify 0.1.0__py3-none-any.whl

agentverify/__init__.py

@@ -0,0 +1,42 @@
+"""agentverify — deterministic testing for AI agent tool call sequences.
+
+Public API re-exports for convenient access::
+
+    from agentverify import (
+        ToolCall, ExecutionResult, TokenUsage,
+        ANY, OrderMode,
+        assert_tool_calls, assert_cost, assert_no_tool_call,
+        assert_final_output, assert_all,
+        LLMCassetteRecorder, CassetteMode,
+    )
+"""
+
+from agentverify.assertions import (
+    assert_all,
+    assert_cost,
+    assert_final_output,
+    assert_no_tool_call,
+    assert_tool_calls,
+)
+from agentverify.cassette.recorder import CassetteMode, LLMCassetteRecorder
+from agentverify.matchers import ANY, OrderMode
+from agentverify.models import ExecutionResult, TokenUsage, ToolCall
+
+__all__ = [
+    # Data models
+    "ToolCall",
+    "ExecutionResult",
+    "TokenUsage",
+    # Matchers
+    "ANY",
+    "OrderMode",
+    # Assertions
+    "assert_tool_calls",
+    "assert_cost",
+    "assert_no_tool_call",
+    "assert_final_output",
+    "assert_all",
+    # Cassette
+    "LLMCassetteRecorder",
+    "CassetteMode",
+]

agentverify/assertions.py

@@ -0,0 +1,324 @@
+"""Assertion engine for agentverify.
+
+Provides assertion functions for verifying tool call sequences,
+cost/token budgets, safety guardrails, and batch assertion execution.
+"""
+
+from __future__ import annotations
+
+from typing import Callable, Optional
+
+from agentverify.errors import (
+    AgentVerifyError,
+    CostBudgetError,
+    FinalOutputError,
+    MultipleAssertionError,
+    SafetyRuleViolationError,
+    ToolCallSequenceError,
+)
+from agentverify.matchers import OrderMode
+from agentverify.models import ExecutionResult, ToolCall
+
+
+def _tool_call_matches(
+    expected: ToolCall,
+    actual: ToolCall,
+    partial_args: bool,
+) -> bool:
+    """Check whether two ToolCalls match.
+
+    Names must match exactly.  Argument matching depends on *partial_args*:
+    - False: arguments must be equal (ANY matcher still works via __eq__)
+    - True:  only keys present in expected.arguments are checked
+    """
+    if expected.name != actual.name:
+        return False
+
+    if partial_args:
+        for key, value in expected.arguments.items():
+            if key not in actual.arguments:
+                return False
+            if value != actual.arguments[key]:
+                return False
+        return True
+
+    return expected.arguments == actual.arguments
+
+
+def assert_tool_calls(
+    result: ExecutionResult,
+    expected: list[ToolCall],
+    order: OrderMode = OrderMode.EXACT,
+    partial_args: bool = False,
+) -> None:
+    """Verify the tool call sequence in an ExecutionResult.
+
+    Args:
+        result: The ExecutionResult to verify.
+        expected: Expected ToolCall list.
+        order: Comparison mode (EXACT, IN_ORDER, ANY_ORDER).
+        partial_args: When True, only keys present in expected arguments
+            are checked against actual arguments.
+
+    Raises:
+        ToolCallSequenceError: When a mismatch is detected.
+    """
+    actual = result.tool_calls
+
+    if order == OrderMode.EXACT:
+        _assert_exact(expected, actual, partial_args)
+    elif order == OrderMode.IN_ORDER:
+        _assert_in_order(expected, actual, partial_args)
+    elif order == OrderMode.ANY_ORDER:
+        _assert_any_order(expected, actual, partial_args)
+    else:  # pragma: no cover — OrderMode is a closed enum
+        pass
+
+
+def _assert_exact(
+    expected: list[ToolCall],
+    actual: list[ToolCall],
+    partial_args: bool,
+) -> None:
+    """EXACT mode: same length and same tool calls at each position."""
+    if len(expected) != len(actual):
+        # first_mismatch_index is the first position that differs,
+        # or min(len(expected), len(actual)) if one list is shorter.
+        mismatch = min(len(expected), len(actual))
+        for i in range(mismatch):
+            if not _tool_call_matches(expected[i], actual[i], partial_args):
+                mismatch = i
+                break
+        raise ToolCallSequenceError(
+            expected=expected,
+            actual=actual,
+            first_mismatch_index=mismatch,
+        )
+
+    for i in range(len(expected)):
+        if not _tool_call_matches(expected[i], actual[i], partial_args):
+            raise ToolCallSequenceError(
+                expected=expected,
+                actual=actual,
+                first_mismatch_index=i,
+            )
+
+
+def _assert_in_order(
+    expected: list[ToolCall],
+    actual: list[ToolCall],
+    partial_args: bool,
+) -> None:
+    """IN_ORDER mode: expected must be a subsequence of actual.
+
+    Uses a two-pointer approach: walk through actual looking for each
+    expected element in order.
+    """
+    if not expected:
+        return
+
+    exp_idx = 0
+    for act_idx in range(len(actual)):
+        if exp_idx < len(expected) and _tool_call_matches(
+            expected[exp_idx], actual[act_idx], partial_args
+        ):
+            exp_idx += 1
+
+    if exp_idx < len(expected):
+        # The first unmatched expected element is the mismatch point
+        raise ToolCallSequenceError(
+            expected=expected,
+            actual=actual,
+            first_mismatch_index=exp_idx,
+        )
+
+
+def _assert_any_order(
+    expected: list[ToolCall],
+    actual: list[ToolCall],
+    partial_args: bool,
+) -> None:
+    """ANY_ORDER mode: all expected tool calls must exist in actual."""
+    for i, exp_tc in enumerate(expected):
+        found = any(
+            _tool_call_matches(exp_tc, act_tc, partial_args)
+            for act_tc in actual
+        )
+        if not found:
+            raise ToolCallSequenceError(
+                expected=expected,
+                actual=actual,
+                first_mismatch_index=i,
+            )
+
+
+def assert_cost(
+    result: ExecutionResult,
+    max_tokens: Optional[int] = None,
+    max_cost_usd: Optional[float] = None,
+    strict: bool = False,
+) -> None:
+    """Verify that cost/token consumption is within budget.
+
+    Args:
+        result: The ExecutionResult to verify.
+        max_tokens: Maximum allowed total tokens.
+        max_cost_usd: Maximum allowed cost in USD.
+        strict: When True, raise CostBudgetError if token_usage or
+            total_cost_usd is None when the corresponding limit is set.
+            Default False silently passes when data is unavailable.
+
+    Raises:
+        CostBudgetError: When budget is exceeded or when strict=True
+            and the required data is missing.
+    """
+    if max_tokens is not None:
+        if result.token_usage is not None:
+            actual_tokens = result.token_usage.total_tokens
+            if actual_tokens > max_tokens:
+                raise CostBudgetError(
+                    actual=actual_tokens,
+                    limit=max_tokens,
+                    exceeded_by=actual_tokens - max_tokens,
+                )
+        elif strict:
+            raise CostBudgetError(
+                actual=0,
+                limit=max_tokens,
+                exceeded_by=0,
+                message="token_usage is None but strict mode requires it",
+            )
+
+    if max_cost_usd is not None:
+        if result.total_cost_usd is not None:
+            if result.total_cost_usd > max_cost_usd:
+                raise CostBudgetError(
+                    actual=result.total_cost_usd,
+                    limit=max_cost_usd,
+                    exceeded_by=result.total_cost_usd - max_cost_usd,
+                )
+        elif strict:
+            raise CostBudgetError(
+                actual=0.0,
+                limit=max_cost_usd,
+                exceeded_by=0.0,
+                message="total_cost_usd is None but strict mode requires it",
+            )
+
+
+def assert_no_tool_call(
+    result: ExecutionResult,
+    forbidden_tools: list[str],
+) -> None:
+    """Verify that no forbidden tools were called.
+
+    All violations are collected and reported at once.
+
+    Args:
+        result: The ExecutionResult to verify.
+        forbidden_tools: List of forbidden tool names.
+
+    Raises:
+        SafetyRuleViolationError: When forbidden tool calls are detected.
+    """
+    forbidden_set = set(forbidden_tools)
+    violations: list[dict] = []
+
+    for position, tc in enumerate(result.tool_calls):
+        if tc.name in forbidden_set:
+            violations.append(
+                {
+                    "tool_name": tc.name,
+                    "arguments": tc.arguments,
+                    "position": position,
+                }
+            )
+
+    if violations:
+        raise SafetyRuleViolationError(violations=violations)
+
+
+def assert_final_output(
+    result: ExecutionResult,
+    contains: Optional[str] = None,
+    equals: Optional[str] = None,
+    matches: Optional[str] = None,
+) -> None:
+    """Verify the agent's final text output.
+
+    At least one of *contains*, *equals*, or *matches* must be provided.
+
+    Args:
+        result: The ExecutionResult to verify.
+        contains: Substring that must appear in final_output.
+        equals: Exact string that final_output must equal.
+        matches: Regex pattern that final_output must match.
+
+    Raises:
+        FinalOutputError: When the final output does not meet expectations.
+    """
+    if contains is None and equals is None and matches is None:
+        raise ValueError(
+            "assert_final_output requires at least one of: contains, equals, matches"
+        )
+
+    if result.final_output is None:
+        raise FinalOutputError("final_output is None")
+
+    if equals is not None:
+        if result.final_output != equals:
+            raise FinalOutputError(
+                f"final_output does not equal expected\n"
+                f"\n"
+                f"  Expected: {equals!r}\n"
+                f"  Actual:   {result.final_output!r}"
+            )
+
+    if contains is not None:
+        if contains not in result.final_output:
+            raise FinalOutputError(
+                f"final_output does not contain expected substring\n"
+                f"\n"
+                f"  Substring: {contains!r}\n"
+                f"  Actual:    {result.final_output!r}"
+            )
+
+    if matches is not None:
+        import re
+
+        if not re.search(matches, result.final_output):
+            raise FinalOutputError(
+                f"final_output does not match pattern\n"
+                f"\n"
+                f"  Pattern: {matches!r}\n"
+                f"  Actual:  {result.final_output!r}"
+            )
+
+
+def assert_all(
+    result: ExecutionResult,
+    *assertions: Callable[[ExecutionResult], None],
+) -> None:
+    """Execute multiple assertions, collecting all failures.
+
+    Each assertion is a callable that takes an ExecutionResult and
+    returns None or raises an AgentVerifyError.  All assertions are
+    executed even if some fail.
+
+    Args:
+        result: The ExecutionResult to verify.
+        *assertions: Callable assertions to execute.
+
+    Raises:
+        MultipleAssertionError: When one or more assertions fail.
+    """
+    errors: list[AgentVerifyError] = []
+
+    for assertion in assertions:
+        try:
+            assertion(result)
+        except AgentVerifyError as exc:
+            errors.append(exc)
+
+    if errors:
+        raise MultipleAssertionError(errors=errors)

agentverify/cassette/__init__.py

@@ -0,0 +1 @@
+

agentverify/cassette/adapters/__init__.py

@@ -0,0 +1 @@
+

agentverify/cassette/adapters/anthropic.py

@@ -0,0 +1,274 @@
+"""Anthropic SDK adapter for LLM cassette recording and replay.
+
+Monkey-patches ``anthropic.resources.messages.Messages.create``
+to intercept message creation calls for recording and replaying via the
+cassette recorder.
+
+anthropic is an **optional** dependency.  Import this module only when the
+anthropic extra is installed (``pip install agentverify[anthropic]``).
+"""
+
+from __future__ import annotations
+
+import json
+from contextlib import contextmanager
+from typing import TYPE_CHECKING, Any, Generator
+from unittest.mock import patch
+
+from agentverify.cassette.adapters.base import (
+    LLMProviderAdapter,
+    NormalizedRequest,
+    NormalizedResponse,
+)
+from agentverify.models import TokenUsage
+
+if TYPE_CHECKING:
+    from agentverify.cassette.recorder import LLMCassetteRecorder
+
+try:
+    import anthropic as _anthropic_module  # noqa: F401 – presence check
+except ImportError:  # pragma: no cover
+    _anthropic_module = None  # type: ignore[assignment]
+
+_PATCH_TARGET = "anthropic.resources.messages.Messages.create"
+
+
+def _ensure_anthropic_installed() -> None:
+    """Raise a clear error when the anthropic package is missing."""
+    if _anthropic_module is None:  # pragma: no cover
+        raise ImportError(
+            "The anthropic package is required to use AnthropicAdapter. "
+            "Install it with: pip install agentverify[anthropic]"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Lightweight response objects used by denormalize_response so that callers
+# can access ``response.content``, ``response.usage``, etc. without needing a
+# real ``anthropic.types.Message`` instance.
+# ---------------------------------------------------------------------------
+
+
+class _Usage:
+    """Minimal stand-in for ``anthropic.types.Usage``."""
+
+    def __init__(self, input_tokens: int, output_tokens: int) -> None:
+        self.input_tokens = input_tokens
+        self.output_tokens = output_tokens
+
+
+class _TextBlock:
+    """Minimal stand-in for ``anthropic.types.TextBlock``."""
+
+    def __init__(self, text: str) -> None:
+        self.type = "text"
+        self.text = text
+
+
+class _ToolUseBlock:
+    """Minimal stand-in for ``anthropic.types.ToolUseBlock``."""
+
+    def __init__(self, id: str, name: str, input: dict[str, Any]) -> None:
+        self.type = "tool_use"
+        self.id = id
+        self.name = name
+        self.input = input
+
+
+class _Message:
+    """Minimal stand-in for ``anthropic.types.Message``."""
+
+    def __init__(
+        self,
+        content: list[_TextBlock | _ToolUseBlock],
+        usage: _Usage | None,
+        stop_reason: str = "end_turn",
+        model: str = "",
+        id: str = "",
+    ) -> None:
+        self.content = content
+        self.usage = usage
+        self.stop_reason = stop_reason
+        self.model = model
+        self.id = id
+
+
+# ---------------------------------------------------------------------------
+# Adapter implementation
+# ---------------------------------------------------------------------------
+
+
+class AnthropicAdapter(LLMProviderAdapter):
+    """Adapter for the Anthropic Python SDK (``anthropic>=0.20``)."""
+
+    @property
+    def name(self) -> str:
+        return "anthropic"
+
+    # -- normalisation -------------------------------------------------------
+
+    def normalize_request(self, raw_request: dict[str, Any]) -> NormalizedRequest:
+        """Convert Anthropic ``messages.create()`` kwargs to *NormalizedRequest*."""
+        messages = raw_request.get("messages", [])
+        model = raw_request.get("model", "")
+
+        # Normalise tools: Anthropic format uses
+        # {"name": ..., "input_schema": {...}}
+        raw_tools = raw_request.get("tools")
+        tools: list[dict[str, Any]] | None = None
+        if raw_tools is not None:
+            tools = []
+            for t in raw_tools:
+                tools.append(
+                    {
+                        "name": t.get("name", ""),
+                        "parameters": t.get("input_schema", {}),
+                    }
+                )
+
+        # Everything else goes into parameters
+        _reserved = {"messages", "model", "tools"}
+        parameters = {k: v for k, v in raw_request.items() if k not in _reserved}
+
+        return NormalizedRequest(
+            messages=messages,
+            model=model,
+            tools=tools,
+            parameters=parameters,
+        )
+
+    def normalize_response(self, raw_response: Any) -> NormalizedResponse:
+        """Convert an Anthropic ``Message`` response to *NormalizedResponse*."""
+        content: str | None = None
+        tool_calls: list[dict[str, Any]] | None = None
+
+        for block in raw_response.content:
+            block_type = getattr(block, "type", None)
+            if block_type == "text":
+                if content is None:
+                    content = block.text
+                else:
+                    content += block.text
+            elif block_type == "tool_use":
+                if tool_calls is None:
+                    tool_calls = []
+                tool_calls.append(
+                    {
+                        "name": block.name,
+                        "arguments": json.dumps(block.input),
+                    }
+                )
+
+        token_usage: TokenUsage | None = None
+        if raw_response.usage is not None:
+            token_usage = TokenUsage(
+                input_tokens=raw_response.usage.input_tokens,
+                output_tokens=raw_response.usage.output_tokens,
+            )
+
+        return NormalizedResponse(
+            content=content,
+            tool_calls=tool_calls,
+            token_usage=token_usage,
+        )
+
+    def denormalize_response(self, normalized: NormalizedResponse) -> Any:
+        """Build a lightweight object that mimics an Anthropic ``Message``."""
+        content_blocks: list[_TextBlock | _ToolUseBlock] = []
+
+        if normalized.content is not None:
+            content_blocks.append(_TextBlock(text=normalized.content))
+
+        if normalized.tool_calls:
+            for i, tc in enumerate(normalized.tool_calls):
+                arguments = tc["arguments"]
+                if isinstance(arguments, str):
+                    try:
+                        arguments = json.loads(arguments)
+                    except (json.JSONDecodeError, ValueError):
+                        arguments = {}
+                content_blocks.append(
+                    _ToolUseBlock(
+                        id=f"toolu_{i}",
+                        name=tc["name"],
+                        input=arguments,
+                    )
+                )
+
+        stop_reason = "tool_use" if normalized.tool_calls else "end_turn"
+
+        usage: _Usage | None = None
+        if normalized.token_usage is not None:
+            usage = _Usage(
+                input_tokens=normalized.token_usage.input_tokens,
+                output_tokens=normalized.token_usage.output_tokens,
+            )
+
+        model = normalized.raw_metadata.get("model", "")
+        message_id = normalized.raw_metadata.get("id", "")
+
+        return _Message(
+            content=content_blocks,
+            usage=usage,
+            stop_reason=stop_reason,
+            model=model,
+            id=message_id,
+        )
+
+    # -- monkey-patching -----------------------------------------------------
+
+    @contextmanager
+    def patch(self, recorder: LLMCassetteRecorder) -> Generator[None, None, None]:
+        """Monkey-patch ``anthropic.resources.messages.Messages.create``.
+
+        In **record** mode the real API is called, the response is normalised,
+        recorded into the cassette, and the original response is returned.
+
+        In **replay** mode the request is normalised, looked up in the
+        cassette, denormalised, and returned without calling the real API.
+        """
+        _ensure_anthropic_installed()
+
+        adapter = self
+
+        def _patched_create(original_fn):  # type: ignore[no-untyped-def]
+            """Return a wrapper that intercepts ``create()`` calls."""
+
+            def wrapper(*args: Any, **kwargs: Any) -> Any:
+                from agentverify.cassette.recorder import CassetteMode, OnMissingRequest
+
+                norm_req = adapter.normalize_request(kwargs)
+
+                if recorder.mode in (CassetteMode.RECORD, CassetteMode.AUTO):
+                    # Call the real API and record the interaction
+                    response = original_fn(*args, **kwargs)
+                    norm_resp = adapter.normalize_response(response)
+                    recorder.record(norm_req, norm_resp)
+                    return response
+
+                # REPLAY mode
+                norm_resp = recorder.lookup(norm_req)
+                if norm_resp is None:
+                    if recorder.on_missing == OnMissingRequest.FALLBACK:
+                        response = original_fn(*args, **kwargs)
+                        norm_resp = adapter.normalize_response(response)
+                        recorder.record(norm_req, norm_resp)
+                        return response
+                    from agentverify.errors import CassetteMissingRequestError
+
+                    raise CassetteMissingRequestError(
+                        f"No matching cassette entry for request: model={norm_req.model}"
+                    )
+                return adapter.denormalize_response(norm_resp)
+
+            return wrapper
+
+        # Use unittest.mock.patch to ensure clean teardown
+        with patch(
+            _PATCH_TARGET,
+            new=_patched_create(
+                # Grab the real function *before* patching
+                __import__("anthropic").resources.messages.Messages.create
+            ),
+        ):
+            yield