contextforge-eval 0.1.0__py3-none-any.whl

context_forge/harness/user_simulator/adapters/crewai.py

@@ -0,0 +1,100 @@
+"""CrewAI adapter for user simulation."""
+
+import asyncio
+from typing import Any
+
+from langchain_core.messages import AIMessage, BaseMessage
+
+from ..models import SimulationState
+
+
+class CrewAIAdapter:
+    """Adapter for CrewAI multi-agent crews.
+
+    Wraps a CrewAI Crew and provides a conversational interface
+    for the simulation harness.
+
+    Note: CrewAI is task-oriented rather than conversational.
+    This adapter treats each user message as a task input.
+
+    Example usage:
+        from crewai import Agent, Crew, Task
+
+        agent = Agent(role="Assistant", goal="Help users", ...)
+        crew = Crew(agents=[agent], tasks=[...])
+
+        adapter = CrewAIAdapter(
+            crew=crew,
+            task_template="User request: {message}",
+        )
+    """
+
+    def __init__(
+        self,
+        crew: Any,
+        task_template: str = "{message}",
+        agent_name: str = "crewai_crew",
+        context_window: int = 5,
+    ):
+        """Initialize CrewAI adapter.
+
+        Args:
+            crew: CrewAI Crew instance
+            task_template: Template for converting messages to tasks
+            agent_name: Name for identification
+            context_window: Number of recent turns to include as context
+        """
+        self._crew = crew
+        self._task_template = task_template
+        self._agent_name = agent_name
+        self._context_window = context_window
+        self._context: list[str] = []
+
+    @property
+    def framework(self) -> str:
+        return "crewai"
+
+    @property
+    def agent_name(self) -> str:
+        return self._agent_name
+
+    async def initialize(self, config: dict[str, Any] | None = None) -> None:
+        """Reset context for new simulation."""
+        self._context = []
+
+    async def invoke(
+        self,
+        message: BaseMessage,
+        state: SimulationState,
+    ) -> BaseMessage:
+        """Invoke CrewAI with user message as task input."""
+        # Format message as task input
+        task_input = self._task_template.format(message=message.content)
+
+        # Build context from recent turns
+        context = "\n".join(self._context[-self._context_window:])
+
+        # Run crew
+        try:
+            result = await asyncio.to_thread(
+                self._crew.kickoff,
+                inputs={"task": task_input, "context": context, "message": message.content}
+            )
+        except Exception as e:
+            # Handle case where crew doesn't accept these inputs
+            result = await asyncio.to_thread(self._crew.kickoff)
+
+        # Store turn for context
+        self._context.append(f"User: {message.content}")
+        result_str = str(result) if result else ""
+        self._context.append(f"Agent: {result_str}")
+
+        return AIMessage(content=result_str)
+
+    async def cleanup(self) -> None:
+        """Clean up CrewAI resources."""
+        pass
+
+    def get_state(self) -> dict[str, Any]:
+        """Return current context state."""
+        return {"context_turns": len(self._context) // 2}

context_forge/harness/user_simulator/adapters/langgraph.py

@@ -0,0 +1,157 @@
+"""LangGraph adapter for user simulation."""
+
+import asyncio
+from typing import Any, Callable, Optional
+
+from langchain_core.messages import AIMessage, BaseMessage
+
+from ..models import SimulationState
+
+
+class LangGraphAdapter:
+    """Adapter for LangGraph StateGraph agents.
+
+    Wraps a compiled LangGraph and translates between the simulation
+    harness message format and LangGraph's state-based invocation.
+
+    Example usage:
+        from my_agent import build_my_graph, MyAgentState
+        from context_forge.instrumentation import LangChainInstrumentor
+
+        graph = build_my_graph()
+
+        # With instrumentation for trace capture
+        instrumentor = LangChainInstrumentor()
+        instrumentor.instrument()
+
+        adapter = LangGraphAdapter(
+            graph=graph,
+            state_class=MyAgentState,
+            input_key="message",
+            output_key="response",
+            callbacks=[instrumentor.get_callback_handler()],
+        )
+    """
+
+    def __init__(
+        self,
+        graph: Any,
+        state_class: Optional[type] = None,
+        input_key: str = "message",
+        output_key: str = "response",
+        messages_key: str = "messages",
+        agent_name: str = "langgraph_agent",
+        initial_state: dict[str, Any] | None = None,
+        config: dict[str, Any] | None = None,
+        state_builder: Optional[Callable[[BaseMessage, SimulationState], dict[str, Any]]] = None,
+        callbacks: list[Any] | None = None,
+    ):
+        """Initialize the LangGraph adapter.
+
+        Args:
+            graph: Compiled LangGraph StateGraph
+            state_class: TypedDict or Pydantic class for agent state (optional)
+            input_key: State key for user input message
+            output_key: State key for agent response
+            messages_key: State key for conversation history
+            agent_name: Name for identification
+            initial_state: Initial state values
+            config: LangGraph config (thread_id, etc.)
+            state_builder: Optional custom function to build input state
+            callbacks: List of callback handlers for instrumentation
+        """
+        self._graph = graph
+        self._state_class = state_class
+        self._input_key = input_key
+        self._output_key = output_key
+        self._messages_key = messages_key
+        self._agent_name = agent_name
+        self._initial_state = initial_state or {}
+        self._config = config or {}
+        self._state_builder = state_builder
+        self._callbacks = callbacks or []
+        self._current_state: dict[str, Any] = {}
+
+    @property
+    def framework(self) -> str:
+        return "langgraph"
+
+    @property
+    def agent_name(self) -> str:
+        return self._agent_name
+
+    async def initialize(self, config: dict[str, Any] | None = None) -> None:
+        """Reset state for a new simulation."""
+        self._current_state = dict(self._initial_state)
+        if config:
+            self._config.update(config)
+
+    async def invoke(
+        self,
+        message: BaseMessage,
+        state: SimulationState,
+    ) -> BaseMessage:
+        """Invoke the LangGraph agent with a user message."""
+        # Build input state
+        if self._state_builder:
+            input_state = self._state_builder(message, state)
+        else:
+            input_state = self._build_default_state(message, state)
+
+        # Invoke graph
+        result = await self._invoke_graph(input_state)
+
+        # Update internal state tracking
+        self._current_state = dict(result)
+
+        # Extract response
+        response_text = result.get(self._output_key, "")
+        if isinstance(response_text, BaseMessage):
+            return response_text
+        return AIMessage(content=str(response_text) if response_text else "")
+
+    def _build_default_state(
+        self,
+        message: BaseMessage,
+        state: SimulationState,
+    ) -> dict[str, Any]:
+        """Build default input state from message and simulation state."""
+        # Get messages from simulation state
+        messages = [t.message for t in state.turns]
+
+        input_state = {
+            self._input_key: message.content,
+            self._messages_key: messages,
+            **self._current_state,
+        }
+
+        # Carry over any fields from initial state that aren't set
+        for key, value in self._initial_state.items():
+            if key not in input_state:
+                input_state[key] = value
+
+        return input_state
+
+    async def _invoke_graph(self, input_state: dict) -> dict:
+        """Invoke the graph, handling sync/async."""
+        # Build config with callbacks for instrumentation
+        invoke_config = dict(self._config)
+        if self._callbacks:
+            # Merge callbacks with existing config callbacks
+            existing_callbacks = invoke_config.get("callbacks", [])
+            invoke_config["callbacks"] = list(existing_callbacks) + list(self._callbacks)
+
+        if hasattr(self._graph, "ainvoke"):
+            return await self._graph.ainvoke(input_state, config=invoke_config)
+        else:
+            return await asyncio.to_thread(
+                self._graph.invoke, input_state, config=invoke_config
+            )
+
+    async def cleanup(self) -> None:
+        """No cleanup needed for LangGraph."""
+        pass
+
+    def get_state(self) -> dict[str, Any]:
+        """Return current agent state."""
+        return dict(self._current_state)

context_forge/harness/user_simulator/adapters/pydanticai.py

@@ -0,0 +1,105 @@
+"""PydanticAI adapter for user simulation."""
+
+import json
+from typing import Any, Callable, Generic, Optional, TypeVar
+
+from langchain_core.messages import AIMessage, BaseMessage
+
+from ..models import SimulationState
+
+T = TypeVar("T")
+
+
+class PydanticAIAdapter(Generic[T]):
+    """Adapter for PydanticAI agents.
+
+    PydanticAI agents use typed dependencies and structured outputs.
+    This adapter manages the dependency injection and conversation state.
+
+    Example usage:
+        from pydantic_ai import Agent
+
+        agent = Agent(
+            model="ollama:llama3.1",
+            system_prompt="You are a helpful assistant.",
+        )
+
+        adapter = PydanticAIAdapter(
+            agent=agent,
+            deps_factory=lambda state: MyDeps(user_id=state.agent_state.get("user_id")),
+        )
+    """
+
+    def __init__(
+        self,
+        agent: Any,
+        deps_factory: Optional[Callable[[SimulationState], T]] = None,
+        agent_name: str = "pydanticai_agent",
+    ):
+        """Initialize PydanticAI adapter.
+
+        Args:
+            agent: PydanticAI Agent instance
+            deps_factory: Factory function to create dependencies from state
+            agent_name: Name for identification
+        """
+        self._agent = agent
+        self._deps_factory = deps_factory
+        self._agent_name = agent_name
+        self._message_history: list[Any] = []
+
+    @property
+    def framework(self) -> str:
+        return "pydanticai"
+
+    @property
+    def agent_name(self) -> str:
+        return self._agent_name
+
+    async def initialize(self, config: dict[str, Any] | None = None) -> None:
+        """Reset for new simulation."""
+        self._message_history = []
+
+    async def invoke(
+        self,
+        message: BaseMessage,
+        state: SimulationState,
+    ) -> BaseMessage:
+        """Invoke PydanticAI agent."""
+        # Create dependencies if factory provided
+        deps = None
+        if self._deps_factory:
+            deps = self._deps_factory(state)
+
+        # Run agent
+        if deps is not None:
+            result = await self._agent.run(
+                message.content,
+                deps=deps,
+                message_history=self._message_history,
+            )
+        else:
+            result = await self._agent.run(
+                message.content,
+                message_history=self._message_history,
+            )
+
+        # Update history
+        if hasattr(result, "all_messages"):
+            self._message_history = result.all_messages()
+
+        # Extract response
+        response_data = result.data if hasattr(result, "data") else str(result)
+        if isinstance(response_data, str):
+            return AIMessage(content=response_data)
+        else:
+            # Structured output - serialize to string
+            return AIMessage(content=json.dumps(response_data, default=str))
+
+    async def cleanup(self) -> None:
+        """Clean up PydanticAI resources."""
+        pass
+
+    def get_state(self) -> dict[str, Any]:
+        """Return current message history state."""
+        return {"message_history_length": len(self._message_history)}

context_forge/harness/user_simulator/llm/__init__.py

@@ -0,0 +1,5 @@
+"""LLM clients for user simulation."""
+
+from .ollama import OllamaClient, OllamaConfig
+
+__all__ = ["OllamaClient", "OllamaConfig"]

context_forge/harness/user_simulator/llm/ollama.py

@@ -0,0 +1,119 @@
+"""Ollama client for user simulation LLM calls."""
+
+from typing import Optional
+
+import httpx
+from pydantic import BaseModel, Field
+
+
+class OllamaConfig(BaseModel):
+    """Configuration for Ollama LLM client."""
+
+    base_url: str = "http://localhost:11434"
+    model: str = "llama3.2"
+    temperature: float = 0.7
+    max_tokens: int = 500
+    timeout: float = 60.0
+
+
+class OllamaClient:
+    """Async client for Ollama API.
+
+    Used by LLMUserSimulator to generate simulated user responses.
+
+    Example usage:
+        async with OllamaClient() as client:
+            response = await client.generate(
+                prompt="What should the user say next?",
+                system="You are simulating a user named Sarah...",
+            )
+    """
+
+    def __init__(self, config: Optional[OllamaConfig] = None):
+        """Initialize the Ollama client.
+
+        Args:
+            config: Configuration for Ollama connection
+        """
+        self._config = config or OllamaConfig()
+        self._client: Optional[httpx.AsyncClient] = None
+
+    async def __aenter__(self) -> "OllamaClient":
+        """Enter async context manager."""
+        self._client = httpx.AsyncClient(timeout=self._config.timeout)
+        return self
+
+    async def __aexit__(self, *args) -> None:
+        """Exit async context manager."""
+        if self._client:
+            await self._client.aclose()
+            self._client = None
+
+    async def generate(
+        self,
+        prompt: str,
+        system: Optional[str] = None,
+    ) -> str:
+        """Generate a response from Ollama.
+
+        Args:
+            prompt: User prompt to send
+            system: Optional system prompt
+
+        Returns:
+            Generated text response
+        """
+        if not self._client:
+            raise RuntimeError("Client not initialized. Use async context manager.")
+
+        messages = []
+        if system:
+            messages.append({"role": "system", "content": system})
+        messages.append({"role": "user", "content": prompt})
+
+        response = await self._client.post(
+            f"{self._config.base_url}/api/chat",
+            json={
+                "model": self._config.model,
+                "messages": messages,
+                "stream": False,
+                "options": {
+                    "temperature": self._config.temperature,
+                    "num_predict": self._config.max_tokens,
+                },
+            },
+        )
+        response.raise_for_status()
+
+        data = response.json()
+        return data["message"]["content"]
+
+    async def check_health(self) -> bool:
+        """Check if Ollama is available.
+
+        Returns:
+            True if Ollama is reachable and responding
+        """
+        if not self._client:
+            raise RuntimeError("Client not initialized. Use async context manager.")
+
+        try:
+            response = await self._client.get(f"{self._config.base_url}/api/tags")
+            return response.status_code == 200
+        except httpx.RequestError:
+            return False
+
+    async def list_models(self) -> list[str]:
+        """List available models.
+
+        Returns:
+            List of model names available in Ollama
+        """
+        if not self._client:
+            raise RuntimeError("Client not initialized. Use async context manager.")
+
+        response = await self._client.get(f"{self._config.base_url}/api/tags")
+        response.raise_for_status()
+
+        data = response.json()
+        return [model["name"] for model in data.get("models", [])]

context_forge/harness/user_simulator/models.py

@@ -0,0 +1,103 @@
+"""Pydantic models for simulation state and results."""
+
+from datetime import datetime
+from enum import Enum
+from typing import Any, Literal, Optional
+
+from langchain_core.messages import BaseMessage
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class ConversationRole(str, Enum):
+    """Role in the conversation."""
+
+    USER = "user"
+    AGENT = "agent"
+    SYSTEM = "system"
+
+
+class SimulationTurn(BaseModel):
+    """Single turn in the simulation conversation."""
+
+    turn_number: int
+    role: ConversationRole
+    message: BaseMessage
+    timestamp: datetime = Field(default_factory=datetime.now)
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+
+class SimulationState(BaseModel):
+    """Current state of a simulation run."""
+
+    simulation_id: str
+    scenario_id: str
+    persona_id: str
+    turns: list[SimulationTurn] = Field(default_factory=list)
+    current_turn: int = 0
+    max_turns: int = 20
+    started_at: datetime = Field(default_factory=datetime.now)
+    ended_at: Optional[datetime] = None
+    status: Literal["running", "completed", "failed", "terminated"] = "running"
+    termination_reason: Optional[str] = None
+    agent_state: dict[str, Any] = Field(default_factory=dict)
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    def get_messages(self) -> list[BaseMessage]:
+        """Get all messages in conversation order."""
+        return [turn.message for turn in self.turns]
+
+    def get_last_agent_message(self) -> Optional[BaseMessage]:
+        """Get the most recent agent message."""
+        for turn in reversed(self.turns):
+            if turn.role == ConversationRole.AGENT:
+                return turn.message
+        return None
+
+    def get_last_user_message(self) -> Optional[BaseMessage]:
+        """Get the most recent user message."""
+        for turn in reversed(self.turns):
+            if turn.role == ConversationRole.USER:
+                return turn.message
+        return None
+
+
+class SimulationResult(BaseModel):
+    """Result of a completed simulation."""
+
+    simulation_id: str
+    state: SimulationState
+    trace_path: Optional[str] = None
+    metrics: dict[str, Any] = Field(default_factory=dict)
+    success: bool = False
+    error: Optional[str] = None
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary for JSON serialization."""
+        return {
+            "simulation_id": self.simulation_id,
+            "scenario_id": self.state.scenario_id,
+            "persona_id": self.state.persona_id,
+            "total_turns": len(self.state.turns),
+            "status": self.state.status,
+            "termination_reason": self.state.termination_reason,
+            "started_at": self.state.started_at.isoformat(),
+            "ended_at": self.state.ended_at.isoformat() if self.state.ended_at else None,
+            "metrics": self.metrics,
+            "success": self.success,
+            "error": self.error,
+            "trace_path": self.trace_path,
+            "conversation": [
+                {
+                    "turn": t.turn_number,
+                    "role": t.role.value,
+                    "content": t.message.content,
+                    "timestamp": t.timestamp.isoformat(),
+                }
+                for t in self.state.turns
+            ],
+        }