checkagent 0.0.1a1__py3-none-any.whl

checkagent/__init__.py

@@ -0,0 +1,94 @@
+"""CheckAgent — The open-source testing framework for AI agents."""
+
+__version__ = "0.0.1a1"
+
+from checkagent.adapters.generic import GenericAdapter, wrap
+from checkagent.conversation.session import Conversation, Turn
+from checkagent.core.config import CheckAgentConfig, load_config
+from checkagent.core.cost import (
+    BudgetExceededError,
+    CostBreakdown,
+    CostReport,
+    CostTracker,
+    calculate_run_cost,
+)
+from checkagent.core.types import (
+    AgentInput,
+    AgentRun,
+    Score,
+    Step,
+    StreamEvent,
+    StreamEventType,
+    ToolCall,
+)
+from checkagent.eval.assertions import (
+    StructuredAssertionError,
+    assert_json_schema,
+    assert_output_matches,
+    assert_output_schema,
+    assert_tool_called,
+)
+from checkagent.mock.fault import FaultInjector
+from checkagent.mock.llm import MatchMode, MockLLM
+from checkagent.mock.mcp import MockMCPServer
+from checkagent.mock.tool import MockTool, literal
+from checkagent.streaming.collector import StreamCollector
+
+_LAZY_ADAPTER_IMPORTS: dict[str, tuple[str, str]] = {
+    "AnthropicAdapter": ("checkagent.adapters.anthropic", "AnthropicAdapter"),
+    "CrewAIAdapter": ("checkagent.adapters.crewai", "CrewAIAdapter"),
+    "LangChainAdapter": ("checkagent.adapters.langchain", "LangChainAdapter"),
+    "OpenAIAgentsAdapter": ("checkagent.adapters.openai_agents", "OpenAIAgentsAdapter"),
+    "PydanticAIAdapter": ("checkagent.adapters.pydantic_ai", "PydanticAIAdapter"),
+}
+
+
+def __getattr__(name: str):
+    """Lazy-load framework adapters to avoid import errors for optional deps."""
+    if name in _LAZY_ADAPTER_IMPORTS:
+        import importlib
+
+        module_path, attr = _LAZY_ADAPTER_IMPORTS[name]
+        mod = importlib.import_module(module_path)
+        return getattr(mod, attr)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+__all__ = [
+    "__version__",
+    "AgentInput",
+    "AgentRun",
+    "AnthropicAdapter",
+    "BudgetExceededError",
+    "calculate_run_cost",
+    "CheckAgentConfig",
+    "CostBreakdown",
+    "CostReport",
+    "CostTracker",
+    "Conversation",
+    "CrewAIAdapter",
+    "FaultInjector",
+    "GenericAdapter",
+    "LangChainAdapter",
+    "MatchMode",
+    "MockLLM",
+    "MockMCPServer",
+    "MockTool",
+    "OpenAIAgentsAdapter",
+    "PydanticAIAdapter",
+    "Score",
+    "Step",
+    "StreamCollector",
+    "StreamEvent",
+    "StreamEventType",
+    "StructuredAssertionError",
+    "ToolCall",
+    "Turn",
+    "assert_json_schema",
+    "assert_output_matches",
+    "assert_output_schema",
+    "assert_tool_called",
+    "literal",
+    "load_config",
+    "wrap",
+]

checkagent/adapters/__init__.py

@@ -0,0 +1,32 @@
+"""Adapters — framework-specific wrappers conforming to AgentAdapter."""
+
+from checkagent.adapters.generic import GenericAdapter, wrap
+
+__all__ = [
+    "AnthropicAdapter",
+    "CrewAIAdapter",
+    "GenericAdapter",
+    "LangChainAdapter",
+    "OpenAIAgentsAdapter",
+    "PydanticAIAdapter",
+    "wrap",
+]
+
+_LAZY_IMPORTS: dict[str, tuple[str, str]] = {
+    "LangChainAdapter": ("checkagent.adapters.langchain", "LangChainAdapter"),
+    "OpenAIAgentsAdapter": ("checkagent.adapters.openai_agents", "OpenAIAgentsAdapter"),
+    "CrewAIAdapter": ("checkagent.adapters.crewai", "CrewAIAdapter"),
+    "PydanticAIAdapter": ("checkagent.adapters.pydantic_ai", "PydanticAIAdapter"),
+    "AnthropicAdapter": ("checkagent.adapters.anthropic", "AnthropicAdapter"),
+}
+
+
+def __getattr__(name: str):
+    """Lazy-load framework adapters to avoid import errors for optional deps."""
+    if name in _LAZY_IMPORTS:
+        module_path, attr = _LAZY_IMPORTS[name]
+        import importlib
+
+        mod = importlib.import_module(module_path)
+        return getattr(mod, attr)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

checkagent/adapters/anthropic.py

@@ -0,0 +1,210 @@
+"""Anthropic Claude SDK adapter — wraps AsyncAnthropic as AgentAdapter.
+
+Requires ``anthropic`` to be installed.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import functools
+import inspect
+import time
+from collections.abc import AsyncIterator
+from typing import Any
+
+from checkagent.core.types import (
+    AgentInput,
+    AgentRun,
+    Step,
+    StreamEvent,
+    StreamEventType,
+    ToolCall,
+)
+
+
+def _ensure_anthropic() -> None:
+    """Raise a clear error if anthropic is not installed."""
+    try:
+        import anthropic  # noqa: F401
+    except ImportError:
+        raise ImportError(
+            "AnthropicAdapter requires anthropic. "
+            "Install it with: pip install anthropic"
+        ) from None
+
+
+def _extract_text(message: Any) -> str:
+    """Extract text content from an Anthropic Message."""
+    content = getattr(message, "content", [])
+    texts: list[str] = []
+    for block in content:
+        if isinstance(block, dict):
+            if block.get("type") == "text":
+                texts.append(block["text"])
+        elif getattr(block, "type", None) == "text" and hasattr(block, "text"):
+            texts.append(block.text)
+    return "\n".join(texts) if texts else ""
+
+
+def _extract_tool_calls(message: Any) -> list[ToolCall]:
+    """Extract tool use blocks from an Anthropic Message."""
+    calls: list[ToolCall] = []
+    content = getattr(message, "content", [])
+    for block in content:
+        if isinstance(block, dict):
+            if block.get("type") == "tool_use":
+                calls.append(ToolCall(
+                    name=block.get("name", "unknown"),
+                    arguments=block.get("input", {}),
+                ))
+        elif getattr(block, "type", None) == "tool_use":
+            calls.append(ToolCall(
+                name=getattr(block, "name", "unknown"),
+                arguments=getattr(block, "input", {}),
+            ))
+    return calls
+
+
+def _extract_token_usage(message: Any) -> dict[str, int | None]:
+    """Extract token usage from an Anthropic Message."""
+    usage_info: dict[str, int | None] = {
+        "prompt_tokens": None,
+        "completion_tokens": None,
+    }
+    usage = getattr(message, "usage", None)
+    if usage:
+        usage_info["prompt_tokens"] = getattr(usage, "input_tokens", None)
+        usage_info["completion_tokens"] = getattr(usage, "output_tokens", None)
+    return usage_info
+
+
+class AnthropicAdapter:
+    """Wraps an Anthropic client as an AgentAdapter."""
+
+    def __init__(
+        self,
+        client: Any,
+        *,
+        model: str = "claude-sonnet-4-20250514",
+        system: str | None = None,
+        max_tokens: int = 1024,
+    ) -> None:
+        _ensure_anthropic()
+        self._client = client
+        self._model = model
+        self._system = system
+        self._max_tokens = max_tokens
+
+    async def run(self, input: AgentInput | str) -> AgentRun:
+        """Send a message and return an AgentRun trace."""
+        if isinstance(input, str):
+            input = AgentInput(query=input)
+
+        kwargs: dict[str, Any] = {
+            "model": self._model,
+            "max_tokens": self._max_tokens,
+            "messages": [{"role": "user", "content": input.query}],
+        }
+        if self._system:
+            kwargs["system"] = self._system
+
+        start = time.perf_counter()
+        try:
+            if hasattr(self._client, "messages") and hasattr(
+                self._client.messages, "create"
+            ):
+                create_fn = self._client.messages.create
+            else:
+                create_fn = self._client.create
+
+            if inspect.iscoroutinefunction(create_fn):
+                message = await create_fn(**kwargs)
+            else:
+                loop = asyncio.get_running_loop()
+                message = await loop.run_in_executor(
+                    None, functools.partial(create_fn, **kwargs)
+                )
+        except Exception as exc:
+            elapsed = (time.perf_counter() - start) * 1000
+            return AgentRun(
+                input=input,
+                error=f"{type(exc).__name__}: {exc}",
+                duration_ms=elapsed,
+            )
+
+        elapsed = (time.perf_counter() - start) * 1000
+        text = _extract_text(message)
+        tool_calls = _extract_tool_calls(message)
+        tokens = _extract_token_usage(message)
+
+        step = Step(
+            step_index=0,
+            input_text=input.query,
+            output_text=text,
+            tool_calls=tool_calls,
+            prompt_tokens=tokens["prompt_tokens"],
+            completion_tokens=tokens["completion_tokens"],
+        )
+
+        return AgentRun(
+            input=input,
+            steps=[step],
+            final_output=message,
+            duration_ms=elapsed,
+            total_prompt_tokens=tokens["prompt_tokens"],
+            total_completion_tokens=tokens["completion_tokens"],
+        )
+
+    async def run_stream(self, input: AgentInput | str) -> AsyncIterator[StreamEvent]:
+        """Stream execution events via Anthropic's streaming API."""
+        if isinstance(input, str):
+            input = AgentInput(query=input)
+
+        kwargs: dict[str, Any] = {
+            "model": self._model,
+            "max_tokens": self._max_tokens,
+            "messages": [{"role": "user", "content": input.query}],
+        }
+        if self._system:
+            kwargs["system"] = self._system
+
+        yield StreamEvent(event_type=StreamEventType.RUN_START)
+
+        try:
+            if hasattr(self._client, "messages") and hasattr(
+                self._client.messages, "stream"
+            ):
+                stream_fn = self._client.messages.stream
+            else:
+                # Fallback: run and synthesize
+                result = await self.run(input)
+                if result.error:
+                    yield StreamEvent(
+                        event_type=StreamEventType.ERROR, data=result.error
+                    )
+                else:
+                    yield StreamEvent(
+                        event_type=StreamEventType.TEXT_DELTA,
+                        data=_extract_text(result.final_output),
+                        step_index=0,
+                    )
+                yield StreamEvent(event_type=StreamEventType.RUN_END, data=result)
+                return
+
+            async with stream_fn(**kwargs) as stream:
+                async for event in stream:
+                    event_type = getattr(event, "type", "")
+                    if event_type == "content_block_delta":
+                        delta = getattr(event, "delta", None)
+                        if delta and getattr(delta, "type", "") == "text_delta":
+                            yield StreamEvent(
+                                event_type=StreamEventType.TEXT_DELTA,
+                                data=getattr(delta, "text", ""),
+                            )
+        except Exception as exc:
+            yield StreamEvent(
+                event_type=StreamEventType.ERROR,
+                data=f"{type(exc).__name__}: {exc}",
+            )
+
+        yield StreamEvent(event_type=StreamEventType.RUN_END)

checkagent/adapters/crewai.py

@@ -0,0 +1,189 @@
+"""CrewAI adapter — wraps Crew.kickoff() as AgentAdapter.
+
+Requires ``crewai`` to be installed. Converts CrewAI's CrewOutput
+into CheckAgent's AgentRun trace format.
+
+Streaming is synthesized from the final result since CrewAI does not
+expose a streaming API for individual steps.
+"""
+
+from __future__ import annotations
+
+import time
+from collections.abc import AsyncIterator
+from typing import Any
+
+from checkagent.core.types import (
+    AgentInput,
+    AgentRun,
+    Step,
+    StreamEvent,
+    StreamEventType,
+    ToolCall,
+)
+
+
+def _ensure_crewai() -> None:
+    """Raise a clear error if crewai is not installed."""
+    try:
+        import crewai  # noqa: F401
+    except ImportError:
+        raise ImportError(
+            "CrewAIAdapter requires crewai. "
+            "Install it with: pip install crewai"
+        ) from None
+
+
+def _extract_steps(result: Any) -> list[Step]:
+    """Extract steps from CrewOutput.tasks_output or agent interactions."""
+    steps: list[Step] = []
+
+    # CrewOutput.tasks_output is a list of TaskOutput objects
+    tasks_output = getattr(result, "tasks_output", None) or []
+    for i, task_out in enumerate(tasks_output):
+        raw_text = getattr(task_out, "raw", "") or ""
+        description = getattr(task_out, "description", "") or ""
+        agent_name = getattr(task_out, "agent", "") or ""
+
+        step = Step(
+            step_index=i,
+            input_text=description,
+            output_text=str(raw_text),
+            metadata={"agent": str(agent_name)} if agent_name else {},
+        )
+        steps.append(step)
+
+    # If no tasks_output, create a single step from the raw result
+    if not steps:
+        raw = getattr(result, "raw", str(result))
+        steps.append(Step(step_index=0, input_text="", output_text=str(raw)))
+
+    return steps
+
+
+def _extract_tool_calls(result: Any) -> list[ToolCall]:
+    """Extract tool calls from CrewOutput if available."""
+    calls: list[ToolCall] = []
+    tasks_output = getattr(result, "tasks_output", None) or []
+    for task_out in tasks_output:
+        # CrewAI TaskOutput may have tool_calls or tools_used
+        tool_calls = getattr(task_out, "tool_calls", None) or []
+        for tc in tool_calls:
+            if isinstance(tc, dict):
+                calls.append(ToolCall(
+                    name=tc.get("name", "unknown"),
+                    arguments=tc.get("arguments", tc.get("args", {})),
+                ))
+            else:
+                calls.append(ToolCall(
+                    name=getattr(tc, "name", "unknown"),
+                    arguments=getattr(tc, "arguments", {}),
+                ))
+    return calls
+
+
+def _extract_token_usage(result: Any) -> dict[str, int | None]:
+    """Extract token usage from CrewOutput.token_usage."""
+    usage: dict[str, int | None] = {
+        "prompt_tokens": None,
+        "completion_tokens": None,
+    }
+    token_usage = getattr(result, "token_usage", None)
+    if token_usage and isinstance(token_usage, dict):
+        usage["prompt_tokens"] = token_usage.get("prompt_tokens")
+        usage["completion_tokens"] = token_usage.get("completion_tokens")
+    elif token_usage:
+        usage["prompt_tokens"] = getattr(token_usage, "prompt_tokens", None)
+        usage["completion_tokens"] = getattr(token_usage, "completion_tokens", None)
+    return usage
+
+
+class CrewAIAdapter:
+    """Wraps a CrewAI Crew as an AgentAdapter.
+
+    Parameters
+    ----------
+    crew : Any
+        A CrewAI ``Crew`` instance.
+    """
+
+    def __init__(self, crew: Any) -> None:
+        _ensure_crewai()
+        self._crew = crew
+
+    async def run(self, input: AgentInput | str) -> AgentRun:
+        """Execute the crew and return an AgentRun trace."""
+        if isinstance(input, str):
+            input = AgentInput(query=input)
+
+        inputs = {"query": input.query, **input.context}
+
+        start = time.perf_counter()
+        try:
+            if hasattr(self._crew, "kickoff_async"):
+                result = await self._crew.kickoff_async(inputs=inputs)
+            else:
+                # Sync-only kickoff — run in executor
+                import asyncio
+                import functools
+
+                loop = asyncio.get_running_loop()
+                result = await loop.run_in_executor(
+                    None, functools.partial(self._crew.kickoff, inputs=inputs)
+                )
+        except Exception as exc:
+            elapsed = (time.perf_counter() - start) * 1000
+            return AgentRun(
+                input=input,
+                error=f"{type(exc).__name__}: {exc}",
+                duration_ms=elapsed,
+            )
+
+        elapsed = (time.perf_counter() - start) * 1000
+        steps = _extract_steps(result)
+        tool_calls = _extract_tool_calls(result)
+        tokens = _extract_token_usage(result)
+
+        # Attach tool calls to the last step if any
+        if tool_calls and steps:
+            steps[-1] = Step(
+                step_index=steps[-1].step_index,
+                input_text=steps[-1].input_text,
+                output_text=steps[-1].output_text,
+                tool_calls=tool_calls,
+                metadata=steps[-1].metadata,
+            )
+
+        final_output = getattr(result, "raw", str(result))
+
+        return AgentRun(
+            input=input,
+            steps=steps,
+            final_output=final_output,
+            duration_ms=elapsed,
+            total_prompt_tokens=tokens["prompt_tokens"],
+            total_completion_tokens=tokens["completion_tokens"],
+        )
+
+    async def run_stream(self, input: AgentInput | str) -> AsyncIterator[StreamEvent]:
+        """Stream execution events (synthesized from final result)."""
+        yield StreamEvent(event_type=StreamEventType.RUN_START)
+
+        try:
+            result = await self.run(input)
+            if result.error:
+                yield StreamEvent(event_type=StreamEventType.ERROR, data=result.error)
+            else:
+                for step in result.steps:
+                    yield StreamEvent(
+                        event_type=StreamEventType.TEXT_DELTA,
+                        data=step.output_text,
+                        step_index=step.step_index,
+                    )
+            yield StreamEvent(event_type=StreamEventType.RUN_END, data=result)
+        except Exception as exc:
+            yield StreamEvent(
+                event_type=StreamEventType.ERROR,
+                data=f"{type(exc).__name__}: {exc}",
+            )
+            yield StreamEvent(event_type=StreamEventType.RUN_END)

checkagent/adapters/generic.py

@@ -0,0 +1,109 @@
+"""Generic adapter — wraps any Python callable as an AgentAdapter.
+
+Supports both sync and async callables. Sync functions are executed
+in a thread pool executor to avoid blocking the event loop.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import functools
+import inspect
+import time
+from collections.abc import AsyncIterator, Callable
+from typing import Any, overload
+
+from checkagent.core.types import AgentInput, AgentRun, Step, StreamEvent, StreamEventType
+
+
+class GenericAdapter:
+    """Wraps any Python callable to conform to the AgentAdapter protocol.
+
+    The callable should accept a string (the query) and return a string
+    or any value that becomes the final_output. Additional kwargs from
+    AgentInput.context are forwarded if the callable accepts **kwargs.
+    """
+
+    def __init__(self, fn: Callable[..., Any]) -> None:
+        self._fn = fn
+        self._is_async = inspect.iscoroutinefunction(fn)
+        self._accepts_kwargs = any(
+            p.kind == inspect.Parameter.VAR_KEYWORD
+            for p in inspect.signature(fn).parameters.values()
+        )
+
+    async def run(self, input: AgentInput | str) -> AgentRun:
+        """Execute the wrapped callable and return an AgentRun trace.
+
+        Accepts either an AgentInput or a plain string (converted automatically).
+        """
+        if isinstance(input, str):
+            input = AgentInput(query=input)
+        start = time.perf_counter()
+        kwargs = input.context if self._accepts_kwargs else {}
+
+        try:
+            if self._is_async:
+                result = await self._fn(input.query, **kwargs)
+            else:
+                loop = asyncio.get_running_loop()
+                result = await loop.run_in_executor(
+                    None, functools.partial(self._fn, input.query, **kwargs)
+                )
+        except Exception as exc:
+            elapsed = (time.perf_counter() - start) * 1000
+            return AgentRun(
+                input=input,
+                error=f"{type(exc).__name__}: {exc}",
+                duration_ms=elapsed,
+            )
+
+        elapsed = (time.perf_counter() - start) * 1000
+        return AgentRun(
+            input=input,
+            steps=[Step(step_index=0, input_text=input.query, output_text=str(result))],
+            final_output=result,
+            duration_ms=elapsed,
+        )
+
+    async def run_stream(self, input: AgentInput) -> AsyncIterator[StreamEvent]:
+        """Fallback stream — runs the callable and synthesizes events."""
+        yield StreamEvent(event_type=StreamEventType.RUN_START)
+
+        result = await self.run(input)
+
+        if result.error:
+            yield StreamEvent(event_type=StreamEventType.ERROR, data=result.error)
+        else:
+            yield StreamEvent(
+                event_type=StreamEventType.TEXT_DELTA,
+                data=str(result.final_output),
+                step_index=0,
+            )
+
+        yield StreamEvent(event_type=StreamEventType.RUN_END, data=result)
+
+
+@overload
+def wrap(fn: Callable[..., Any]) -> GenericAdapter: ...
+
+
+@overload
+def wrap() -> Callable[[Callable[..., Any]], GenericAdapter]: ...
+
+
+def wrap(
+    fn: Callable[..., Any] | None = None,
+) -> GenericAdapter | Callable[[Callable[..., Any]], GenericAdapter]:
+    """Wrap a callable as a GenericAdapter. Usable as decorator or function.
+
+    @wrap
+    async def my_agent(query: str) -> str:
+        ...
+
+    # or
+    adapter = wrap(my_sync_function)
+    """
+    if fn is not None:
+        return GenericAdapter(fn)
+    return GenericAdapter  # type: ignore[return-value]