ragbits-evaluate 0.5.0__py3-none-any.whl → 1.4.0.dev202602030301__py3-none-any.whl

ragbits/evaluate/agent_simulation/__init__.py

@@ -0,0 +1,87 @@
+"""Agent simulation utilities for evaluation scenarios.
+
+This module uses lazy imports for components that require optional dependencies
+(ragbits-agents, ragbits-chat) to allow importing result models independently.
+"""
+
+from typing import TYPE_CHECKING
+
+# Import context, metrics, and result models eagerly - they have no external dependencies
+from ragbits.evaluate.agent_simulation.context import DataSnapshot, DomainContext
+from ragbits.evaluate.agent_simulation.metrics import (
+    CompositeMetricCollector,
+    LatencyMetricCollector,
+    MetricCollector,
+    TokenUsageMetricCollector,
+    ToolUsageMetricCollector,
+)
+from ragbits.evaluate.agent_simulation.results import (
+    ConversationMetrics,
+    SimulationResult,
+    SimulationStatus,
+    TaskResult,
+    TurnResult,
+)
+
+if TYPE_CHECKING:
+    from ragbits.evaluate.agent_simulation.conversation import run_simulation
+    from ragbits.evaluate.agent_simulation.deepeval_evaluator import DeepEvalEvaluator
+    from ragbits.evaluate.agent_simulation.logger import ConversationLogger
+    from ragbits.evaluate.agent_simulation.models import Personality, Scenario, Task, Turn
+    from ragbits.evaluate.agent_simulation.scenarios import load_personalities, load_scenarios
+    from ragbits.evaluate.agent_simulation.simulation import GoalChecker, SimulatedUser
+
+__all__ = [
+    "CompositeMetricCollector",
+    "ConversationLogger",
+    "ConversationMetrics",
+    "DataSnapshot",
+    "DeepEvalEvaluator",
+    "DomainContext",
+    "GoalChecker",
+    "LatencyMetricCollector",
+    "MetricCollector",
+    "Personality",
+    "Scenario",
+    "SimulatedUser",
+    "SimulationResult",
+    "SimulationStatus",
+    "Task",
+    "TaskResult",
+    "TokenUsageMetricCollector",
+    "ToolUsageMetricCollector",
+    "Turn",
+    "TurnResult",
+    "load_personalities",
+    "load_scenarios",
+    "run_simulation",
+]
+
+
+def __getattr__(name: str) -> object:
+    """Lazy import for components with optional dependencies."""
+    if name == "run_simulation":
+        from ragbits.evaluate.agent_simulation.conversation import run_simulation
+
+        return run_simulation
+    if name == "DeepEvalEvaluator":
+        from ragbits.evaluate.agent_simulation.deepeval_evaluator import DeepEvalEvaluator
+
+        return DeepEvalEvaluator
+    if name == "ConversationLogger":
+        from ragbits.evaluate.agent_simulation.logger import ConversationLogger
+
+        return ConversationLogger
+    if name in ("Personality", "Scenario", "Task", "Turn"):
+        from ragbits.evaluate.agent_simulation import models
+
+        return getattr(models, name)
+    if name in ("load_personalities", "load_scenarios"):
+        from ragbits.evaluate.agent_simulation import scenarios
+
+        return getattr(scenarios, name)
+    if name in ("GoalChecker", "SimulatedUser"):
+        from ragbits.evaluate.agent_simulation import simulation
+
+        return getattr(simulation, name)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

ragbits/evaluate/agent_simulation/context.py

@@ -0,0 +1,118 @@
+"""Context models for agent simulation scenarios."""
+
+from dataclasses import dataclass, field
+from typing import Any
+
+DEFAULT_MAX_ITEMS_IN_PROMPT = 15
+
+
+@dataclass
+class DomainContext:
+    """Domain-specific context for goal checking and simulation.
+
+    Provides additional context to the GoalChecker to avoid false negatives
+    from value interpretation differences or missing domain knowledge.
+
+    The context is intentionally generic - use the `metadata` field for any
+    domain-specific information that doesn't fit the standard fields.
+
+    Example:
+        >>> context = DomainContext(
+        ...     domain_type="customer_support",
+        ...     locale="en_US",
+        ...     metadata={"ticket_statuses": ["open", "pending", "resolved"]},
+        ... )
+        >>> result = await goal_checker.is_task_achieved(task, history, context=context)
+    """
+
+    domain_type: str
+    """Type of domain (e.g., "customer_support", "booking", "search", "qa")."""
+
+    locale: str = "en_US"
+    """Locale for language and formatting (e.g., "en_US", "de_DE")."""
+
+    metadata: dict[str, Any] = field(default_factory=dict)
+    """Arbitrary domain-specific metadata for goal checking context."""
+
+    def format_for_prompt(self) -> str:
+        """Format context for inclusion in LLM prompts.
+
+        Returns:
+            Formatted string suitable for prompt injection.
+        """
+        parts = [
+            f"Domain: {self.domain_type}",
+            f"Locale: {self.locale}",
+        ]
+
+        if self.metadata:
+            parts.append("Additional context:")
+            for key, value in self.metadata.items():
+                if isinstance(value, list) and len(value) > DEFAULT_MAX_ITEMS_IN_PROMPT:
+                    truncated = value[:DEFAULT_MAX_ITEMS_IN_PROMPT]
+                    parts.append(f"  {key}: {truncated} ... and {len(value) - DEFAULT_MAX_ITEMS_IN_PROMPT} more")
+                else:
+                    parts.append(f"  {key}: {value}")
+
+        return "\n".join(parts)
+
+
+@dataclass
+class DataSnapshot:
+    """Sample of available data to ground simulated user requests.
+
+    Provides the simulated user with knowledge of what data actually exists,
+    preventing unrealistic requests for non-existent entities.
+
+    The snapshot is intentionally generic - store any domain-specific data
+    in the `entities` dict with descriptive keys.
+
+    Example:
+        >>> snapshot = DataSnapshot(
+        ...     entities={
+        ...         "available_topics": ["billing", "technical", "returns"],
+        ...         "sample_users": [{"id": "u1", "name": "John"}],
+        ...     },
+        ...     description="Customer support knowledge base",
+        ... )
+        >>> # SimulatedUser will only reference items from this data
+    """
+
+    entities: dict[str, list[Any]] = field(default_factory=dict)
+    """Named collections of available entities (e.g., {"users": [...], "documents": [...]})."""
+
+    description: str = ""
+    """Optional description of the data snapshot for context."""
+
+    def format_for_prompt(self, max_items: int = DEFAULT_MAX_ITEMS_IN_PROMPT) -> str:
+        """Format data snapshot for inclusion in LLM prompts.
+
+        Args:
+            max_items: Maximum number of items to include per entity type.
+
+        Returns:
+            Formatted string suitable for prompt injection.
+        """
+        parts = []
+
+        if self.description:
+            parts.append(f"Context: {self.description}")
+
+        for entity_name, entity_list in self.entities.items():
+            if not entity_list:
+                continue
+
+            truncated = entity_list[:max_items]
+            # Format items - if dicts with 'name', use that; otherwise str()
+            formatted_items = []
+            for item in truncated:
+                if isinstance(item, dict) and "name" in item:
+                    formatted_items.append(item["name"])
+                else:
+                    formatted_items.append(str(item))
+
+            parts.append(f"{entity_name}: {', '.join(formatted_items)}")
+            if len(entity_list) > max_items:
+                parts.append(f"  ... and {len(entity_list) - max_items} more")
+
+        return "\n".join(parts)

ragbits/evaluate/agent_simulation/conversation.py

@@ -0,0 +1,333 @@
+"""Conversation orchestration for agent simulation scenarios."""
+
+from datetime import datetime, timezone
+
+from ragbits.agents.tool import ToolCallResult
+from ragbits.chat.interface import ChatInterface
+from ragbits.chat.interface.types import ChatContext
+from ragbits.core.llms import Usage
+from ragbits.evaluate.agent_simulation.context import DataSnapshot, DomainContext
+from ragbits.evaluate.agent_simulation.deepeval_evaluator import DeepEvalEvaluator
+from ragbits.evaluate.agent_simulation.logger import ConversationLogger
+from ragbits.evaluate.agent_simulation.metrics.collectors import CompositeMetricCollector, MetricCollector
+from ragbits.evaluate.agent_simulation.models import Personality, Scenario, Turn
+from ragbits.evaluate.agent_simulation.results import (
+    ConversationMetrics,
+    SimulationResult,
+    SimulationStatus,
+    TaskResult,
+    TurnResult,
+)
+from ragbits.evaluate.agent_simulation.simulation import GoalChecker, SimulatedUser, ToolUsageChecker, build_llm
+
+
+def _evaluate_with_deepeval(history: list[Turn], logger: ConversationLogger) -> dict[str, float]:
+    """Evaluate conversation with DeepEval metrics.
+
+    Args:
+        history: List of conversation turns to evaluate
+        logger: Logger instance to record evaluation results
+
+    Returns:
+        Dictionary of metric names to scores
+    """
+    scores: dict[str, float] = {}
+    if not history:
+        return scores
+
+    print("\n=== Running DeepEval Evaluation ===")
+    deepeval_evaluator = DeepEvalEvaluator()
+    try:
+        evaluation_results = deepeval_evaluator.evaluate_conversation(history)
+        logger.log_deepeval_metrics(evaluation_results)
+        for metric_name, result in evaluation_results.items():
+            score = result.get("score")
+            if score is not None:
+                print(f"{metric_name}: {score:.4f}")
+                scores[metric_name] = float(score)
+            else:
+                error = result.get("error", "Unknown error")
+                print(f"{metric_name}: Error - {error}")
+    except Exception as e:
+        print(f"Error during DeepEval evaluation: {e}")
+        logger.log_deepeval_metrics({"error": str(e)})
+
+    return scores
+
+
+async def run_simulation(  # noqa: PLR0912, PLR0915
+    scenario: Scenario,
+    chat: ChatInterface,
+    max_turns_scenario: int = 15,
+    max_turns_task: int | None = 4,
+    log_file: str | None = None,
+    agent_model_name: str | None = None,
+    sim_user_model_name: str | None = None,
+    checker_model_name: str | None = None,
+    default_model: str = "gpt-4o-mini",
+    api_key: str = "",
+    user_message_prefix: str = "",
+    personality: Personality | None = None,
+    domain_context: DomainContext | None = None,
+    data_snapshot: DataSnapshot | None = None,
+    metric_collectors: list[MetricCollector] | None = None,
+) -> SimulationResult:
+    """Run a conversation between an agent and a simulated user.
+
+    The conversation proceeds through tasks defined in the scenario, with a goal checker
+    determining when each task is completed. Returns structured results for programmatic access.
+
+    Args:
+        scenario: The scenario containing tasks to complete
+        chat: An instantiated ChatInterface used to drive the assistant side of the conversation.
+        max_turns_scenario: Maximum number of conversation turns for the entire scenario
+        max_turns_task: Maximum number of conversation turns per task (None for no limit)
+        log_file: Optional path to log file
+        agent_model_name: Optional override for agent LLM model name
+        sim_user_model_name: Optional override for simulated user LLM model name
+        checker_model_name: Optional override for goal checker LLM model name
+        default_model: Default LLM model name
+        api_key: API key for LLM
+        user_message_prefix: Optional prefix to add to user messages before sending to agent
+        personality: Optional personality to use for the simulated user
+        domain_context: Optional domain context for goal checking (currency, locale, business rules)
+        data_snapshot: Optional data snapshot to ground simulated user requests to available data
+        metric_collectors: Optional list of custom metric collectors for per-turn metrics
+
+    Returns:
+        SimulationResult containing all turns, task results, and metrics
+    """
+    start_time = datetime.now(timezone.utc)
+
+    # Initialize metric collectors
+    collectors = CompositeMetricCollector(metric_collectors)
+
+    # Initialize result tracking
+    turn_results: list[TurnResult] = []
+    task_results: list[TaskResult] = []
+    task_turn_counts: dict[int, int] = {}  # task_index -> turns taken
+    task_final_reasons: dict[int, str] = {}  # task_index -> final reason
+
+    # Simulated user uses an independent llm (can share the same provider)
+    sim_user = SimulatedUser(
+        llm=build_llm(sim_user_model_name, default_model, api_key),
+        scenario=scenario,
+        personality=personality,
+        data_snapshot=data_snapshot,
+    )
+    # Independent goal checker model
+    goal_checker = GoalChecker(llm=build_llm(checker_model_name, default_model, api_key), scenario=scenario)
+    # Tool usage checker (simple comparator, no LLM needed)
+    tool_checker = ToolUsageChecker(scenario=scenario)
+
+    history: list[Turn] = []
+    logger = ConversationLogger(log_file)
+    logger.initialize_session(scenario, agent_model_name, sim_user_model_name, checker_model_name, personality)
+    total_usage = Usage()
+
+    # Seed: ask the simulated user for the first message based on the first task
+    user_message = await sim_user.next_message(history=history)
+
+    turns_for_current_task = 0
+    current_task_id = None
+    status = SimulationStatus.RUNNING
+    error_message: str | None = None
+
+    try:
+        for turn_idx in range(1, max_turns_scenario + 1):
+            current_task = sim_user.get_current_task()
+            if current_task is None:
+                print("\nAll tasks completed!")
+                status = SimulationStatus.COMPLETED
+                break
+
+            current_task_index = sim_user.current_task_idx
+
+            # Track turns per task
+            if current_task_id != id(current_task):
+                # New task started, reset counter
+                turns_for_current_task = 0
+                current_task_id = id(current_task)
+
+            # Check if we've exceeded max turns for this task
+            if max_turns_task is not None and turns_for_current_task >= max_turns_task:
+                print(f"\nReached maximum number of turns ({max_turns_task}) for current task. Exiting.")
+                status = SimulationStatus.TIMEOUT
+                task_final_reasons[current_task_index] = f"Timeout: exceeded {max_turns_task} turns"
+                break
+
+            turns_for_current_task += 1
+            task_turn_counts[current_task_index] = task_turn_counts.get(current_task_index, 0) + 1
+
+            print(f"\n=== Turn {turn_idx} (Task turn: {turns_for_current_task}) ===")
+            print(f"Current Task: {current_task.task}")
+            print(f"User: {user_message}")
+
+            # Notify metric collectors of turn start
+            collectors.on_turn_start(turn_idx, current_task_index, user_message)
+
+            # Add optional prefix to user message
+            full_user_message = user_message_prefix + user_message if user_message_prefix else user_message
+
+            assistant_reply_parts: list[str] = []
+            tool_calls: list[ToolCallResult] = []
+            turn_usage: Usage = Usage()
+            dummy_chat_context = ChatContext()
+
+            stream = chat.chat(
+                message=full_user_message,
+                history=[
+                    d
+                    for turn in history
+                    for d in ({"role": "user", "text": turn.user}, {"role": "assistant", "text": turn.assistant})
+                ],
+                context=dummy_chat_context,
+            )
+
+            async for chunk in stream:
+                if isinstance(chunk, str):
+                    assistant_reply_parts.append(chunk)
+                elif isinstance(chunk, ToolCallResult):
+                    tool_calls.append(chunk)
+                elif isinstance(chunk, Usage):
+                    turn_usage += chunk
+
+            total_usage += turn_usage
+            assistant_reply = "".join(assistant_reply_parts).strip()
+            print(f"Assistant: {assistant_reply}")
+            if tool_calls:
+                print(f"Tools used: {[tc.name for tc in tool_calls]}")
+            print(
+                f"Assistant token usage: {turn_usage.total_tokens} total "
+                f"({turn_usage.prompt_tokens} prompt + {turn_usage.completion_tokens} completion), "
+                f"estimated cost: ${turn_usage.estimated_cost:.6f}"
+            )
+            logger.log_turn(turn_idx, current_task, user_message, assistant_reply, tool_calls, turn_usage)
+
+            # Update simulation-visible history (Turn objects)
+            history.append(Turn(user=user_message, assistant=assistant_reply))
+
+            # Ask the judge if current task is achieved
+            task_done, reason = await goal_checker.is_task_achieved(current_task, history, context=domain_context)
+            logger.log_task_check(turn_idx, task_done, reason)
+
+            # Check tool usage if expected tools are specified
+            if current_task.expected_tools:
+                tools_used_correctly, tool_reason = tool_checker.check_tool_usage(current_task, tool_calls)
+                logger.log_tool_check(turn_idx, tools_used_correctly, tool_reason, tool_calls)
+                if not tools_used_correctly:
+                    print(f"Tool usage issue: {tool_reason}")
+                else:
+                    print(f"Tool usage verified: {tool_reason}")
+
+            # Create turn result
+            turn_result = TurnResult(
+                turn_index=turn_idx,
+                task_index=current_task_index,
+                user_message=user_message,
+                assistant_message=assistant_reply,
+                tool_calls=[{"name": tc.name, "arguments": tc.arguments, "result": tc.result} for tc in tool_calls],
+                task_completed=task_done,
+                task_completed_reason=reason,
+                token_usage={
+                    "total": turn_usage.total_tokens,
+                    "prompt": turn_usage.prompt_tokens,
+                    "completion": turn_usage.completion_tokens,
+                },
+            )
+            turn_results.append(turn_result)
+
+            # Notify metric collectors of turn end
+            collectors.on_turn_end(turn_result)
+
+            if task_done:
+                print(f"Task completed: {reason}")
+                task_final_reasons[current_task_index] = reason
+                has_next = sim_user.advance_to_next_task()
+                if not has_next:
+                    print("\nAll tasks completed!")
+                    status = SimulationStatus.COMPLETED
+                    break
+                next_task = sim_user.get_current_task()
+                if next_task:
+                    logger.log_task_transition(next_task)
+                # Reset task turn counter when moving to next task
+                turns_for_current_task = 0
+                current_task_id = id(next_task) if next_task else None
+            else:
+                print(f"Task not completed: {reason}")
+                task_final_reasons[current_task_index] = reason
+
+            # Ask the simulator for the next user message
+            user_message = await sim_user.next_message(history)
+
+        else:
+            print("\nReached maximum number of turns. Exiting.")
+            status = SimulationStatus.TIMEOUT
+
+    except Exception as e:
+        status = SimulationStatus.FAILED
+        error_message = str(e)
+        print(f"\nSimulation failed with error: {error_message}")
+
+    # Build task results
+    for i, task in enumerate(scenario.tasks):
+        completed = i < sim_user.current_task_idx or (
+            i == sim_user.current_task_idx and status == SimulationStatus.COMPLETED
+        )
+        task_results.append(
+            TaskResult(
+                task_index=i,
+                description=task.task,
+                expected_result=task.expected_result,
+                completed=completed,
+                turns_taken=task_turn_counts.get(i, 0),
+                final_reason=task_final_reasons.get(i, "Not attempted"),
+            )
+        )
+
+    # Print total token usage summary
+    print("\n=== Total Assistant Token Usage ===")
+    print(
+        f"Total assistant tokens: {total_usage.total_tokens} "
+        f"({total_usage.prompt_tokens} prompt + {total_usage.completion_tokens} completion)"
+    )
+    print(f"Total estimated cost: ${total_usage.estimated_cost:.6f}")
+    logger.log_total_usage(total_usage)
+
+    # Evaluate conversation with DeepEval metrics
+    deepeval_scores = _evaluate_with_deepeval(history, logger)
+
+    # Collect custom metrics from collectors
+    custom_metrics = collectors.on_conversation_end(turn_results)
+
+    logger.finalize_session()
+
+    # Build metrics
+    tasks_completed = sum(1 for t in task_results if t.completed)
+    metrics = ConversationMetrics(
+        total_turns=len(turn_results),
+        total_tasks=len(scenario.tasks),
+        tasks_completed=tasks_completed,
+        total_tokens=total_usage.total_tokens,
+        prompt_tokens=total_usage.prompt_tokens,
+        completion_tokens=total_usage.completion_tokens,
+        total_cost_usd=total_usage.estimated_cost,
+        deepeval_scores=deepeval_scores,
+        custom=custom_metrics,
+    )
+
+    return SimulationResult(
+        scenario_name=scenario.name,
+        start_time=start_time,
+        end_time=datetime.now(timezone.utc),
+        status=status,
+        agent_model=agent_model_name,
+        simulated_user_model=sim_user_model_name,
+        checker_model=checker_model_name,
+        personality=personality.name if personality else None,
+        error=error_message,
+        turns=turn_results,
+        tasks=task_results,
+        metrics=metrics,
+    )

ragbits/evaluate/agent_simulation/deepeval_evaluator.py

@@ -0,0 +1,92 @@
+"""DeepEval integration for multi-turn conversation evaluation."""
+
+from deepeval.metrics import (  # type: ignore[attr-defined]
+    ConversationCompletenessMetric,
+    ConversationRelevancyMetric,
+    KnowledgeRetentionMetric,
+)
+from deepeval.test_case import ConversationalTestCase, LLMTestCase  # type: ignore[attr-defined]
+
+from ragbits.evaluate.agent_simulation.models import Turn
+
+
+class DeepEvalEvaluator:
+    """Evaluator using DeepEval metrics for multi-turn conversations."""
+
+    def __init__(self) -> None:
+        """Initialize the DeepEval evaluator with multi-turn metrics."""
+        self.completeness_metric = ConversationCompletenessMetric()
+        self.knowledge_retention_metric = KnowledgeRetentionMetric()
+        self.conversation_relevancy_metric = ConversationRelevancyMetric()
+
+    @staticmethod
+    def _evaluate_metric(
+        metric: ConversationCompletenessMetric | KnowledgeRetentionMetric | ConversationRelevancyMetric,
+        test_case: ConversationalTestCase,
+    ) -> dict[str, float | str | None]:
+        """Evaluate a single metric on a test case.
+
+        Args:
+            metric: The metric instance to evaluate
+            test_case: The conversational test case to evaluate
+
+        Returns:
+            Dictionary containing score, reason, success, and optionally error
+        """
+        try:
+            metric.measure(test_case)
+            return {
+                "score": metric.score,
+                "reason": getattr(metric, "reason", None),
+                "success": getattr(metric, "success", None),
+            }
+        except Exception as e:
+            return {
+                "score": None,
+                "reason": None,
+                "success": None,
+                "error": str(e),
+            }
+
+    def _evaluate_all_metrics(self, test_case: ConversationalTestCase) -> dict[str, dict[str, float | str | None]]:
+        """Evaluate all metrics on a test case.
+
+        Args:
+            test_case: The conversational test case to evaluate
+
+        Returns:
+            Dictionary containing evaluation results for each metric
+        """
+        results: dict[str, dict[str, float | str | None]] = {}
+
+        results["ConversationCompletenessMetric"] = self._evaluate_metric(self.completeness_metric, test_case)
+
+        results["KnowledgeRetentionMetric"] = self._evaluate_metric(self.knowledge_retention_metric, test_case)
+
+        results["ConversationRelevancyMetric"] = self._evaluate_metric(self.conversation_relevancy_metric, test_case)
+
+        return results
+
+    def evaluate_conversation(self, turns: list[Turn]) -> dict[str, dict[str, float | str | None]]:
+        """Evaluate a conversation using DeepEval metrics.
+
+        Args:
+            turns: List of conversation turns to evaluate
+
+        Returns:
+            Dictionary containing evaluation results for each metric
+        """
+        if not turns:
+            return {}
+
+        # Convert ragbits Turn objects to deepeval LLMTestCase objects
+        deepeval_turns = []
+        for turn in turns:
+            # Each turn becomes an LLMTestCase where input is user message and actual_output is assistant response
+            deepeval_turns.append(LLMTestCase(input=turn.user, actual_output=turn.assistant))
+
+        # Create conversational test case
+        test_case = ConversationalTestCase(turns=deepeval_turns)
+
+        # Evaluate with each metric
+        return self._evaluate_all_metrics(test_case)