vanna 0.7.9__py3-none-any.whl → 2.0.0rc1__py3-none-any.whl

vanna/core/errors.py

@@ -0,0 +1,47 @@
+"""
+Exception classes for the Vanna Agents framework.
+
+This module defines all custom exceptions used throughout the framework.
+"""
+
+
+class AgentError(Exception):
+    """Base exception for agent framework."""
+
+    pass
+
+
+class ToolExecutionError(AgentError):
+    """Error during tool execution."""
+
+    pass
+
+
+class ToolNotFoundError(AgentError):
+    """Tool not found in registry."""
+
+    pass
+
+
+class PermissionError(AgentError):
+    """User lacks required permissions."""
+
+    pass
+
+
+class ConversationNotFoundError(AgentError):
+    """Conversation not found."""
+
+    pass
+
+
+class LlmServiceError(AgentError):
+    """Error communicating with LLM service."""
+
+    pass
+
+
+class ValidationError(AgentError):
+    """Data validation error."""
+
+    pass

vanna/core/evaluation/__init__.py

@@ -0,0 +1,81 @@
+"""
+Evaluation framework for Vanna Agents.
+
+This module provides a complete evaluation system for testing and comparing
+agent variants, with special focus on LLM comparison use cases.
+
+Key Features:
+- Parallel execution for efficient I/O-bound operations
+- Multiple built-in evaluators (trajectory, output, LLM-as-judge, efficiency)
+- Rich reporting (HTML, CSV, console)
+- Dataset loaders (YAML, JSON)
+- Agent variant comparison
+
+Example:
+    >>> from vanna.evaluation import (
+    ...     EvaluationRunner,
+    ...     EvaluationDataset,
+    ...     AgentVariant,
+    ...     TrajectoryEvaluator,
+    ...     OutputEvaluator,
+    ... )
+    >>>
+    >>> # Load test dataset
+    >>> dataset = EvaluationDataset.from_yaml("tests/sql_tasks.yaml")
+    >>>
+    >>> # Create agent variants
+    >>> variants = [
+    ...     AgentVariant("claude", claude_agent),
+    ...     AgentVariant("gpt", gpt_agent),
+    ... ]
+    >>>
+    >>> # Run comparison
+    >>> runner = EvaluationRunner(
+    ...     evaluators=[TrajectoryEvaluator(), OutputEvaluator()],
+    ...     max_concurrency=20
+    ... )
+    >>> comparison = await runner.compare_agents(variants, dataset.test_cases)
+    >>> comparison.print_summary()
+"""
+
+from .base import (
+    Evaluator,
+    TestCase,
+    ExpectedOutcome,
+    AgentResult,
+    EvaluationResult,
+    TestCaseResult,
+    AgentVariant,
+)
+from .runner import EvaluationRunner
+from .evaluators import (
+    TrajectoryEvaluator,
+    OutputEvaluator,
+    LLMAsJudgeEvaluator,
+    EfficiencyEvaluator,
+)
+from .report import EvaluationReport, ComparisonReport
+from .dataset import EvaluationDataset
+
+__all__ = [
+    # Base classes
+    "Evaluator",
+    "TestCase",
+    "ExpectedOutcome",
+    "AgentResult",
+    "EvaluationResult",
+    "TestCaseResult",
+    "AgentVariant",
+    # Runner
+    "EvaluationRunner",
+    # Built-in evaluators
+    "TrajectoryEvaluator",
+    "OutputEvaluator",
+    "LLMAsJudgeEvaluator",
+    "EfficiencyEvaluator",
+    # Reporting
+    "EvaluationReport",
+    "ComparisonReport",
+    # Datasets
+    "EvaluationDataset",
+]

vanna/core/evaluation/base.py

@@ -0,0 +1,186 @@
+"""
+Core evaluation abstractions for the Vanna Agents framework.
+
+This module provides the base classes and models for evaluating agent behavior,
+including test cases, expected outcomes, and evaluation results.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional, Callable
+from dataclasses import dataclass, field
+from datetime import datetime
+from pydantic import BaseModel
+
+from vanna.core import User, UiComponent
+
+
+class ExpectedOutcome(BaseModel):
+    """Defines what we expect from the agent for a test case.
+
+    Provides multiple ways to specify expectations:
+    - tools_called: List of tool names that should be called
+    - tools_not_called: List of tool names that should NOT be called
+    - final_answer_contains: Keywords/phrases that should appear in output
+    - final_answer_not_contains: Keywords/phrases that should NOT appear
+    - min_components: Minimum number of UI components expected
+    - max_execution_time_ms: Maximum allowed execution time
+    - custom_validators: Custom validation functions
+    """
+
+    tools_called: Optional[List[str]] = None
+    tools_not_called: Optional[List[str]] = None
+    final_answer_contains: Optional[List[str]] = None
+    final_answer_not_contains: Optional[List[str]] = None
+    min_components: Optional[int] = None
+    max_components: Optional[int] = None
+    max_execution_time_ms: Optional[float] = None
+    metadata: Dict[str, Any] = {}
+
+
+class TestCase(BaseModel):
+    """A single evaluation test case.
+
+    Attributes:
+        id: Unique identifier for the test case
+        user: User context for the test
+        message: The message to send to the agent
+        conversation_id: Optional conversation ID for multi-turn tests
+        expected_outcome: What we expect the agent to do/produce
+        metadata: Additional metadata for categorization/filtering
+    """
+
+    id: str
+    user: User
+    message: str
+    conversation_id: Optional[str] = None
+    expected_outcome: Optional[ExpectedOutcome] = None
+    metadata: Dict[str, Any] = {}
+
+
+@dataclass
+class AgentResult:
+    """The result of running an agent on a test case.
+
+    Captures everything that happened during agent execution
+    for later evaluation.
+    """
+
+    test_case_id: str
+    components: List[UiComponent]
+    tool_calls: List[Dict[str, Any]] = field(default_factory=list)
+    llm_requests: List[Dict[str, Any]] = field(default_factory=list)
+    execution_time_ms: float = 0.0
+    total_tokens: int = 0
+    error: Optional[str] = None
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+    def get_final_answer(self) -> str:
+        """Extract the final answer from components."""
+        # Find text components and concatenate
+        texts = []
+        for component in self.components:
+            if hasattr(component, "rich_component"):
+                rich_comp = component.rich_component
+                if hasattr(rich_comp, "type") and rich_comp.type.value == "text":
+                    content = rich_comp.data.get("content") or getattr(
+                        rich_comp, "content", ""
+                    )
+                    if content:
+                        texts.append(content)
+        return "\n".join(texts)
+
+    def get_tool_names_called(self) -> List[str]:
+        """Get list of tool names that were called."""
+        return [call.get("tool_name", "") for call in self.tool_calls]
+
+
+class EvaluationResult(BaseModel):
+    """Result of evaluating a single test case.
+
+    Attributes:
+        test_case_id: ID of the test case evaluated
+        evaluator_name: Name of the evaluator that produced this result
+        passed: Whether the test case passed
+        score: Score from 0.0 to 1.0
+        reasoning: Explanation of the evaluation
+        metrics: Additional metrics captured during evaluation
+        timestamp: When the evaluation was performed
+    """
+
+    test_case_id: str
+    evaluator_name: str
+    passed: bool
+    score: float  # 0.0 to 1.0
+    reasoning: str
+    metrics: Dict[str, Any] = {}
+    timestamp: datetime = datetime.now()
+
+
+@dataclass
+class TestCaseResult:
+    """Complete result for a single test case including all evaluations."""
+
+    test_case: TestCase
+    agent_result: AgentResult
+    evaluations: List[EvaluationResult]
+    execution_time_ms: float
+
+    def overall_passed(self) -> bool:
+        """Check if all evaluations passed."""
+        return all(e.passed for e in self.evaluations)
+
+    def overall_score(self) -> float:
+        """Calculate average score across all evaluations."""
+        if not self.evaluations:
+            return 0.0
+        return sum(e.score for e in self.evaluations) / len(self.evaluations)
+
+
+@dataclass
+class AgentVariant:
+    """A variant of an agent to evaluate (different LLM, config, etc).
+
+    Used for comparing different agent configurations, especially
+    different LLMs or model versions.
+
+    Attributes:
+        name: Human-readable name for this variant
+        agent: The agent instance to evaluate
+        metadata: Additional info (model name, provider, config, etc)
+    """
+
+    name: str
+    agent: Any  # Agent type - avoiding circular import
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+class Evaluator(ABC):
+    """Base class for evaluating agent behavior.
+
+    Evaluators examine the agent's execution and determine if it
+    met expectations. Multiple evaluators can be composed to check
+    different aspects (trajectory, output quality, efficiency, etc).
+    """
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Name of this evaluator."""
+        pass
+
+    @abstractmethod
+    async def evaluate(
+        self,
+        test_case: TestCase,
+        agent_result: AgentResult,
+    ) -> EvaluationResult:
+        """Evaluate a single test case execution.
+
+        Args:
+            test_case: The test case that was executed
+            agent_result: The result from running the agent
+
+        Returns:
+            EvaluationResult with pass/fail, score, and reasoning
+        """
+        pass

vanna/core/evaluation/dataset.py

@@ -0,0 +1,254 @@
+"""
+Dataset loaders for evaluation test cases.
+
+This module provides utilities for loading test case datasets from
+YAML and JSON files.
+"""
+
+import json
+import yaml
+from typing import Any, Dict, List
+from pathlib import Path
+
+from .base import TestCase, ExpectedOutcome
+from vanna.core import User
+
+
+class EvaluationDataset:
+    """Collection of test cases with metadata.
+
+    Example YAML format:
+        dataset:
+          name: "SQL Generation Tasks"
+          description: "Test cases for SQL generation"
+          test_cases:
+            - id: "sql_001"
+              user_id: "test_user"
+              message: "Show me total sales by region"
+              expected_outcome:
+                tools_called: ["generate_sql", "execute_query"]
+                final_answer_contains: ["SELECT", "GROUP BY", "region"]
+    """
+
+    def __init__(self, name: str, test_cases: List[TestCase], description: str = ""):
+        """Initialize evaluation dataset.
+
+        Args:
+            name: Name of the dataset
+            test_cases: List of test cases
+            description: Optional description
+        """
+        self.name = name
+        self.test_cases = test_cases
+        self.description = description
+
+    @classmethod
+    def from_yaml(cls, path: str) -> "EvaluationDataset":
+        """Load dataset from YAML file.
+
+        Args:
+            path: Path to YAML file
+
+        Returns:
+            EvaluationDataset instance
+        """
+        with open(path, "r") as f:
+            data = yaml.safe_load(f)
+
+        return cls._from_dict(data)
+
+    @classmethod
+    def from_json(cls, path: str) -> "EvaluationDataset":
+        """Load dataset from JSON file.
+
+        Args:
+            path: Path to JSON file
+
+        Returns:
+            EvaluationDataset instance
+        """
+        with open(path, "r") as f:
+            data = json.load(f)
+
+        return cls._from_dict(data)
+
+    @classmethod
+    def _from_dict(cls, data: Dict[str, Any]) -> "EvaluationDataset":
+        """Create dataset from dictionary.
+
+        Args:
+            data: Dictionary with dataset structure
+
+        Returns:
+            EvaluationDataset instance
+        """
+        dataset_config = data.get("dataset", data)
+        name = dataset_config.get("name", "Unnamed Dataset")
+        description = dataset_config.get("description", "")
+
+        test_cases = []
+        for tc_data in dataset_config.get("test_cases", []):
+            test_case = cls._parse_test_case(tc_data)
+            test_cases.append(test_case)
+
+        return cls(name=name, test_cases=test_cases, description=description)
+
+    @classmethod
+    def _parse_test_case(cls, data: Dict[str, Any]) -> TestCase:
+        """Parse a single test case from dictionary.
+
+        Args:
+            data: Test case dictionary
+
+        Returns:
+            TestCase instance
+        """
+        # Create user
+        user_id = data.get("user_id", "test_user")
+        user = User(
+            id=user_id,
+            username=data.get("username", user_id),
+            email=data.get("email", f"{user_id}@example.com"),
+            group_memberships=data.get("user_groups", []),
+        )
+
+        # Parse expected outcome if present
+        expected_outcome = None
+        if "expected_outcome" in data:
+            outcome_data = data["expected_outcome"]
+            expected_outcome = ExpectedOutcome(
+                tools_called=outcome_data.get("tools_called"),
+                tools_not_called=outcome_data.get("tools_not_called"),
+                final_answer_contains=outcome_data.get("final_answer_contains"),
+                final_answer_not_contains=outcome_data.get("final_answer_not_contains"),
+                min_components=outcome_data.get("min_components"),
+                max_components=outcome_data.get("max_components"),
+                max_execution_time_ms=outcome_data.get("max_execution_time_ms"),
+                metadata=outcome_data.get("metadata", {}),
+            )
+
+        return TestCase(
+            id=data["id"],
+            user=user,
+            message=data["message"],
+            conversation_id=data.get("conversation_id"),
+            expected_outcome=expected_outcome,
+            metadata=data.get("metadata", {}),
+        )
+
+    def save_yaml(self, path: str) -> None:
+        """Save dataset to YAML file.
+
+        Args:
+            path: Path to save YAML file
+        """
+        data = self._to_dict()
+        with open(path, "w") as f:
+            yaml.dump(data, f, default_flow_style=False, sort_keys=False)
+
+    def save_json(self, path: str) -> None:
+        """Save dataset to JSON file.
+
+        Args:
+            path: Path to save JSON file
+        """
+        data = self._to_dict()
+        with open(path, "w") as f:
+            json.dump(data, f, indent=2)
+
+    def _to_dict(self) -> Dict[str, Any]:
+        """Convert dataset to dictionary.
+
+        Returns:
+            Dictionary representation
+        """
+        return {
+            "dataset": {
+                "name": self.name,
+                "description": self.description,
+                "test_cases": [self._test_case_to_dict(tc) for tc in self.test_cases],
+            }
+        }
+
+    def _test_case_to_dict(self, test_case: TestCase) -> Dict[str, Any]:
+        """Convert test case to dictionary.
+
+        Args:
+            test_case: TestCase to convert
+
+        Returns:
+            Dictionary representation
+        """
+        data: Dict[str, Any] = {
+            "id": test_case.id,
+            "user_id": test_case.user.id,
+            "username": test_case.user.username,
+            "email": test_case.user.email,
+            "user_groups": test_case.user.group_memberships,
+            "message": test_case.message,
+        }
+
+        if test_case.conversation_id:
+            data["conversation_id"] = test_case.conversation_id
+
+        if test_case.expected_outcome:
+            outcome = test_case.expected_outcome
+            outcome_dict: Dict[str, Any] = {}
+
+            if outcome.tools_called:
+                outcome_dict["tools_called"] = outcome.tools_called
+            if outcome.tools_not_called:
+                outcome_dict["tools_not_called"] = outcome.tools_not_called
+            if outcome.final_answer_contains:
+                outcome_dict["final_answer_contains"] = outcome.final_answer_contains
+            if outcome.final_answer_not_contains:
+                outcome_dict["final_answer_not_contains"] = (
+                    outcome.final_answer_not_contains
+                )
+            if outcome.min_components is not None:
+                outcome_dict["min_components"] = outcome.min_components
+            if outcome.max_components is not None:
+                outcome_dict["max_components"] = outcome.max_components
+            if outcome.max_execution_time_ms is not None:
+                outcome_dict["max_execution_time_ms"] = outcome.max_execution_time_ms
+            if outcome.metadata:
+                outcome_dict["metadata"] = outcome.metadata
+
+            if outcome_dict:
+                data["expected_outcome"] = outcome_dict
+
+        if test_case.metadata:
+            data["metadata"] = test_case.metadata
+
+        return data
+
+    def filter_by_metadata(self, **kwargs: Any) -> "EvaluationDataset":
+        """Filter test cases by metadata fields.
+
+        Args:
+            **kwargs: Metadata fields to match
+
+        Returns:
+            New EvaluationDataset with filtered test cases
+        """
+        filtered = [
+            tc
+            for tc in self.test_cases
+            if all(tc.metadata.get(k) == v for k, v in kwargs.items())
+        ]
+
+        return EvaluationDataset(
+            name=f"{self.name} (filtered)",
+            test_cases=filtered,
+            description=f"Filtered from: {self.description}",
+        )
+
+    def __len__(self) -> int:
+        """Get number of test cases."""
+        return len(self.test_cases)
+
+    def __repr__(self) -> str:
+        """String representation."""
+        return (
+            f"EvaluationDataset(name='{self.name}', test_cases={len(self.test_cases)})"
+        )