ragbits-evaluate 0.5.0__py3-none-any.whl → 1.4.0.dev202602030301__py3-none-any.whl

ragbits/evaluate/agent_simulation/results.py

@@ -0,0 +1,200 @@
+"""Result models for agent simulation scenarios."""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import Any
+
+
+class SimulationStatus(str, Enum):
+    """Status of a simulation run."""
+
+    RUNNING = "running"
+    COMPLETED = "completed"
+    FAILED = "failed"
+    TIMEOUT = "timeout"
+
+
+@dataclass
+class TurnResult:
+    """Result of a single conversation turn."""
+
+    turn_index: int
+    task_index: int
+    user_message: str
+    assistant_message: str
+    tool_calls: list[dict[str, Any]] = field(default_factory=list)
+    task_completed: bool = False
+    task_completed_reason: str = ""
+    token_usage: dict[str, int] | None = None
+    latency_ms: float | None = None
+
+
+@dataclass
+class TaskResult:
+    """Result of a single task within the scenario."""
+
+    task_index: int
+    description: str
+    expected_result: str | None
+    completed: bool
+    turns_taken: int
+    final_reason: str
+
+
+@dataclass
+class ConversationMetrics:
+    """Aggregate metrics for the conversation."""
+
+    total_turns: int
+    total_tasks: int
+    tasks_completed: int
+    total_tokens: int = 0
+    prompt_tokens: int = 0
+    completion_tokens: int = 0
+    total_cost_usd: float = 0.0
+    deepeval_scores: dict[str, float] = field(default_factory=dict)
+    custom: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def success_rate(self) -> float:
+        """Calculate task success rate."""
+        if self.total_tasks == 0:
+            return 0.0
+        return self.tasks_completed / self.total_tasks
+
+
+@dataclass
+class SimulationResult:
+    """Complete result for a scenario simulation."""
+
+    # Metadata
+    scenario_name: str
+    start_time: datetime
+    status: SimulationStatus
+
+    # Detailed data
+    turns: list[TurnResult] = field(default_factory=list)
+    tasks: list[TaskResult] = field(default_factory=list)
+    metrics: ConversationMetrics | None = None
+
+    # Optional metadata
+    end_time: datetime | None = None
+    agent_model: str | None = None
+    simulated_user_model: str | None = None
+    checker_model: str | None = None
+    personality: str | None = None
+    error: str | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to JSON-serializable dictionary."""
+        return {
+            "scenario_name": self.scenario_name,
+            "start_time": self.start_time.isoformat(),
+            "end_time": self.end_time.isoformat() if self.end_time else None,
+            "status": self.status.value,
+            "agent_model": self.agent_model,
+            "simulated_user_model": self.simulated_user_model,
+            "checker_model": self.checker_model,
+            "personality": self.personality,
+            "error": self.error,
+            "turns": [
+                {
+                    "turn_index": t.turn_index,
+                    "task_index": t.task_index,
+                    "user_message": t.user_message,
+                    "assistant_message": t.assistant_message,
+                    "tool_calls": t.tool_calls,
+                    "task_completed": t.task_completed,
+                    "task_completed_reason": t.task_completed_reason,
+                    "token_usage": t.token_usage,
+                    "latency_ms": t.latency_ms,
+                }
+                for t in self.turns
+            ],
+            "tasks": [
+                {
+                    "task_index": t.task_index,
+                    "description": t.description,
+                    "expected_result": t.expected_result,
+                    "completed": t.completed,
+                    "turns_taken": t.turns_taken,
+                    "final_reason": t.final_reason,
+                }
+                for t in self.tasks
+            ],
+            "metrics": {
+                "total_turns": self.metrics.total_turns,
+                "total_tasks": self.metrics.total_tasks,
+                "tasks_completed": self.metrics.tasks_completed,
+                "success_rate": self.metrics.success_rate,
+                "total_tokens": self.metrics.total_tokens,
+                "prompt_tokens": self.metrics.prompt_tokens,
+                "completion_tokens": self.metrics.completion_tokens,
+                "total_cost_usd": self.metrics.total_cost_usd,
+                "deepeval_scores": self.metrics.deepeval_scores,
+                "custom": self.metrics.custom,
+            }
+            if self.metrics
+            else None,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "SimulationResult":
+        """Create from dictionary."""
+        turns = [
+            TurnResult(
+                turn_index=t["turn_index"],
+                task_index=t["task_index"],
+                user_message=t["user_message"],
+                assistant_message=t["assistant_message"],
+                tool_calls=t.get("tool_calls", []),
+                task_completed=t.get("task_completed", False),
+                task_completed_reason=t.get("task_completed_reason", ""),
+                token_usage=t.get("token_usage"),
+                latency_ms=t.get("latency_ms"),
+            )
+            for t in data.get("turns", [])
+        ]
+
+        tasks = [
+            TaskResult(
+                task_index=t["task_index"],
+                description=t["description"],
+                expected_result=t.get("expected_result"),
+                completed=t["completed"],
+                turns_taken=t["turns_taken"],
+                final_reason=t["final_reason"],
+            )
+            for t in data.get("tasks", [])
+        ]
+
+        metrics_data = data.get("metrics")
+        metrics = None
+        if metrics_data:
+            metrics = ConversationMetrics(
+                total_turns=metrics_data["total_turns"],
+                total_tasks=metrics_data["total_tasks"],
+                tasks_completed=metrics_data["tasks_completed"],
+                total_tokens=metrics_data.get("total_tokens", 0),
+                prompt_tokens=metrics_data.get("prompt_tokens", 0),
+                completion_tokens=metrics_data.get("completion_tokens", 0),
+                total_cost_usd=metrics_data.get("total_cost_usd", 0.0),
+                deepeval_scores=metrics_data.get("deepeval_scores", {}),
+                custom=metrics_data.get("custom", {}),
+            )
+
+        return cls(
+            scenario_name=data["scenario_name"],
+            start_time=datetime.fromisoformat(data["start_time"]),
+            end_time=datetime.fromisoformat(data["end_time"]) if data.get("end_time") else None,
+            status=SimulationStatus(data["status"]),
+            agent_model=data.get("agent_model"),
+            simulated_user_model=data.get("simulated_user_model"),
+            checker_model=data.get("checker_model"),
+            personality=data.get("personality"),
+            error=data.get("error"),
+            turns=turns,
+            tasks=tasks,
+            metrics=metrics,
+        )

ragbits/evaluate/agent_simulation/scenarios.py

@@ -0,0 +1,129 @@
+"""Scenario loading functionality for agent simulation."""
+
+import json
+from pathlib import Path
+
+from ragbits.evaluate.agent_simulation.models import Personality, Scenario, Task
+
+
+def load_scenarios(scenarios_file: str = "scenarios.json") -> list[Scenario]:
+    """Load scenarios from a JSON file.
+
+    Expected JSON format:
+    [
+      {
+        "name": "Scenario 1",
+        "tasks": [
+          {
+            "task": "task description",
+            "expected_result": "expected result description",
+            "expected_tools": ["tool1", "tool2"]  # optional
+          },
+          ...
+        ]
+      },
+      ...
+    ]
+
+    Args:
+        scenarios_file: Path to the JSON file containing scenarios
+
+    Returns:
+        List of Scenario objects
+
+    Raises:
+        FileNotFoundError: If the scenarios file doesn't exist
+        ValueError: If the file format is invalid
+    """
+    scenarios_path = Path(scenarios_file)
+    if not scenarios_path.exists():
+        raise FileNotFoundError(f"Scenarios file not found: {scenarios_path}")
+
+    with scenarios_path.open("r", encoding="utf-8") as f:
+        data = json.load(f)
+
+    if not isinstance(data, list):
+        raise ValueError(f"Scenarios file must contain a JSON array, got {type(data).__name__}")
+
+    scenarios: list[Scenario] = []
+    for scenario_data in data:
+        if not isinstance(scenario_data, dict):
+            raise ValueError(f"Each scenario must be a JSON object, got {type(scenario_data).__name__}")
+
+        name = scenario_data.get("name", "")
+        tasks_data = scenario_data.get("tasks", [])
+
+        if not isinstance(tasks_data, list):
+            raise ValueError(f"Tasks must be a JSON array, got {type(tasks_data).__name__}")
+
+        tasks: list[Task] = []
+        for task_data in tasks_data:
+            if not isinstance(task_data, dict):
+                raise ValueError(f"Each task must be a JSON object, got {type(task_data).__name__}")
+
+            task_desc = task_data.get("task", "")
+            expected_result = task_data.get("expected_result", "")
+            expected_tools = task_data.get("expected_tools")
+            if expected_tools is not None and not isinstance(expected_tools, list):
+                raise ValueError(f"expected_tools must be a list or null, got {type(expected_tools).__name__}")
+            tasks.append(Task(task=task_desc, expected_result=expected_result, expected_tools=expected_tools))
+
+        scenarios.append(Scenario(name=name, tasks=tasks))
+
+    if not scenarios:
+        raise ValueError(f"No scenarios found in {scenarios_path}")
+
+    return scenarios
+
+
+def load_personalities(personalities_file: str = "personalities.json") -> list[Personality]:
+    """Load personalities from a JSON file.
+
+    Expected JSON format:
+    [
+      {
+        "name": "Personality 1",
+        "description": "Personality description that will be used in the system prompt"
+      },
+      ...
+    ]
+
+    Args:
+        personalities_file: Path to the JSON file containing personalities
+
+    Returns:
+        List of Personality objects
+
+    Raises:
+        FileNotFoundError: If the personalities file doesn't exist
+        ValueError: If the file format is invalid
+    """
+    personalities_path = Path(personalities_file)
+    if not personalities_path.exists():
+        raise FileNotFoundError(f"Personalities file not found: {personalities_path}")
+
+    with personalities_path.open("r", encoding="utf-8") as f:
+        data = json.load(f)
+
+    if not isinstance(data, list):
+        raise ValueError(f"Personalities file must contain a JSON array, got {type(data).__name__}")
+
+    personalities: list[Personality] = []
+    for personality_data in data:
+        if not isinstance(personality_data, dict):
+            raise ValueError(f"Each personality must be a JSON object, got {type(personality_data).__name__}")
+
+        name = personality_data.get("name", "")
+        description = personality_data.get("description", "")
+
+        if not name:
+            raise ValueError("Each personality must have a 'name' field")
+        if not description:
+            raise ValueError("Each personality must have a 'description' field")
+
+        personalities.append(Personality(name=name, description=description))
+
+    if not personalities:
+        raise ValueError(f"No personalities found in {personalities_path}")
+
+    return personalities

ragbits/evaluate/agent_simulation/simulation.py

@@ -0,0 +1,243 @@
+"""Simulation components for agent evaluation scenarios."""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import TYPE_CHECKING
+
+from ragbits.agents.tool import ToolCallResult
+from ragbits.core.llms import LiteLLM
+from ragbits.evaluate.agent_simulation.models import Personality, Scenario, Task, Turn
+
+if TYPE_CHECKING:
+    from ragbits.evaluate.agent_simulation.context import DataSnapshot, DomainContext
+
+
+class SimulatedUser:
+    """A simple LLM-driven user simulator that works through tasks sequentially.
+
+    It generates the next user utterance based on the conversation so far
+    and the current task. It only moves to the next task when the current one is completed.
+
+    Supports optional data grounding via DataSnapshot to ensure the simulated user
+    only requests items that actually exist in the available data.
+    """
+
+    def __init__(
+        self,
+        llm: LiteLLM,
+        scenario: Scenario,
+        personality: Personality | None = None,
+        data_snapshot: DataSnapshot | None = None,
+    ) -> None:
+        """Initialize the simulated user.
+
+        Args:
+            llm: The LLM to use for generating messages.
+            scenario: The scenario containing tasks to work through.
+            personality: Optional personality to influence communication style.
+            data_snapshot: Optional data snapshot for grounding requests to available data.
+        """
+        self.llm = llm
+        self.scenario = scenario
+        self.personality = personality
+        self.data_snapshot = data_snapshot
+        self.current_task_idx = 0
+
+    def get_current_task(self) -> Task | None:
+        """Get the current task, or None if all tasks are completed."""
+        if self.current_task_idx < len(self.scenario.tasks):
+            return self.scenario.tasks[self.current_task_idx]
+        return None
+
+    def advance_to_next_task(self) -> bool:
+        """Move to the next task. Returns True if there is a next task, False otherwise."""
+        if self.current_task_idx < len(self.scenario.tasks) - 1:
+            self.current_task_idx += 1
+            return True
+        return False
+
+    async def next_message(self, history: list[Turn]) -> str:
+        """Generate the next user message based on conversation history and current task."""
+        current_task = self.get_current_task()
+        if current_task is None:
+            return "Thank you, all tasks are completed."
+
+        history_text = []
+        for t in history:
+            history_text.append(f"User: {t.user}\nAssistant: {t.assistant}")
+        history_block = "\n\n".join(history_text) if history_text else "(no prior messages)"
+
+        task_context = f"Current task: {current_task.task}"
+        if self.current_task_idx > 0:
+            completed_tasks = ", ".join([t.task for t in self.scenario.tasks[: self.current_task_idx]])
+            task_context += f"\nCompleted tasks: {completed_tasks}"
+
+        personality_instruction = ""
+        if self.personality:
+            personality_instruction = f"\n\nPersonality: {self.personality.description}"
+
+        # Build data grounding block if snapshot is provided
+        grounding_block = ""
+        if self.data_snapshot:
+            grounding_block = (
+                "\n\n[AVAILABLE DATA]\n"
+                f"{self.data_snapshot.format_for_prompt()}\n\n"
+                "IMPORTANT: Only reference items that exist in the AVAILABLE DATA above. "
+                "Do not ask for entities that are not listed."
+            )
+
+        prompt = (
+            "[SYSTEM]\n"
+            "You are simulating a concise human user in a terminal chat. "
+            f"Scenario: {self.scenario.name}\n"
+            f"{task_context}{personality_instruction}{grounding_block}\n"
+            "Given the assistant's last reply and the conversation so far, "
+            "write ONLY the next user message to work on the current task. Be specific and brief.\n\n"
+            "[CONVERSATION]\n"
+            f"{history_block}\n\n"
+            "[TASK]\nWrite the next USER message now:"
+        )
+
+        response = await self.llm.generate(prompt=prompt)
+        return response.strip()
+
+
+class GoalChecker:
+    """A lightweight judge model that decides whether the current task has been achieved.
+
+    It inspects the conversation so far and checks if the task matches the expected result.
+    Supports optional domain context for accurate evaluation in specific domains.
+    """
+
+    def __init__(self, llm: LiteLLM, scenario: Scenario) -> None:
+        self.llm = llm
+        self.scenario = scenario
+
+    async def is_task_achieved(
+        self,
+        current_task: Task,
+        history: list[Turn],
+        context: DomainContext | None = None,
+    ) -> tuple[bool, str]:
+        """Check if the current task has been completed based on the conversation history.
+
+        Args:
+            current_task: The task to check completion for.
+            history: List of conversation turns so far.
+            context: Optional domain context for accurate evaluation (e.g., currency, locale).
+
+        Returns:
+            Tuple of (is_completed, reason).
+        """
+        history_text = []
+        for t in history:
+            history_text.append(f"User: {t.user}\nAssistant: {t.assistant}")
+        history_block = "\n\n".join(history_text) if history_text else "(no prior messages)"
+
+        # Build context block if provided
+        context_block = ""
+        if context:
+            context_block = (
+                "\n[IMPORTANT CONTEXT]\n"
+                f"{context.format_for_prompt()}\n\n"
+                "When evaluating task completion, consider the domain context above "
+                f"and use {context.locale} locale conventions.\n\n"
+            )
+
+        prompt = (
+            "[SYSTEM]\n"
+            "You are a strict task-completion judge for a user-assistant conversation. "
+            "Decide if the assistant has fulfilled the current task.\n"
+            f"Current task: {current_task.task}\n"
+            f"Expected result: {current_task.expected_result}\n"
+            f"{context_block}"
+            "Respond with a concise JSON object ONLY, no extra text, with fields:\n"
+            '{"done": true|false, "reason": "short reason"}\n\n'
+            "[CONVERSATION]\n"
+            f"{history_block}\n\n"
+            "[TASK]\nReturn the JSON now:"
+        )
+
+        response = await self.llm.generate(prompt=prompt)
+        text = response.strip()
+        # Be robust to slight deviations by attempting a minimal parse
+        done = False
+        reason = ""
+
+        if not text:
+            return False, "Empty response from goal checker"
+
+        # Try to extract JSON from markdown code blocks if present
+        code_block_match = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", text, re.DOTALL)
+        if code_block_match:
+            text = code_block_match.group(1)
+
+        # Try to find JSON object in the text
+        json_match = re.search(r"\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}", text, re.DOTALL)
+        if json_match:
+            text = json_match.group(0)
+
+        try:
+            data = json.loads(text)
+            done = bool(data.get("done", False))
+            reason = str(data.get("reason", "")).strip()
+        except json.JSONDecodeError:
+            # If JSON parsing fails, try to infer from response text
+            reason = f"Failed to parse JSON response: {text[:100]}"
+            # Heuristic: if response contains "done" or "completed" or "true", assume done
+            text_lower = text.lower()
+            if any(word in text_lower for word in ["done", "completed", "true", "yes", "success"]):
+                done = True
+            elif any(word in text_lower for word in ["not done", "incomplete", "false", "no", "failed"]):
+                done = False
+
+        return done, reason
+
+
+class ToolUsageChecker:
+    """A simple comparator that verifies whether the agent used the expected tools.
+
+    It checks if all expected tools from the task were called during the conversation turn.
+    """
+
+    def __init__(self, scenario: Scenario) -> None:
+        self.scenario = scenario
+
+    def check_tool_usage(self, current_task: Task, tool_calls: list[ToolCallResult]) -> tuple[bool, str]:  # noqa: PLR6301
+        """Check if the expected tools were used for the current task.
+
+        Args:
+            current_task: The current task being evaluated
+            tool_calls: List of tool calls made during this turn
+
+        Returns:
+            Tuple of (success: bool, reason: str)
+        """
+        if not current_task.expected_tools:
+            return True, "No expected tools specified"
+
+        if not tool_calls:
+            return False, "No tools were called, but tools were expected"
+
+        # Get the names of tools that were actually called
+        called_tool_names = [tc.name for tc in tool_calls]
+        expected_tool_names = current_task.expected_tools
+
+        # Check if all expected tools were used
+        missing_tools = [tool for tool in expected_tool_names if tool not in called_tool_names]
+
+        if missing_tools:
+            return (
+                False,
+                f"Expected tools not used: {', '.join(missing_tools)}. Tools called: {', '.join(called_tool_names)}",
+            )
+
+        return True, f"All expected tools used: {', '.join(called_tool_names)}"
+
+
+def build_llm(model_name: str | None, default_model: str, api_key: str) -> LiteLLM:
+    """Build an LLM instance with the specified model name or default."""
+    model = model_name or default_model
+    return LiteLLM(model_name=model, use_structured_output=True, api_key=api_key)