vanna 0.7.9__py3-none-any.whl → 2.0.0rc1__py3-none-any.whl

vanna/core/evaluation/runner.py

@@ -0,0 +1,313 @@
+"""
+Evaluation runner with parallel execution support.
+
+This module provides the EvaluationRunner class that executes test cases
+against agents with configurable parallelism for efficient evaluation,
+especially when comparing multiple LLMs or model versions.
+"""
+
+import asyncio
+from typing import Any, List, Dict, Optional, AsyncGenerator, TYPE_CHECKING
+from datetime import datetime
+
+from .base import (
+    TestCase,
+    AgentResult,
+    TestCaseResult,
+    AgentVariant,
+    Evaluator,
+)
+from vanna.core import UiComponent
+from vanna.core.user.request_context import RequestContext
+from vanna.core.observability import ObservabilityProvider
+
+if TYPE_CHECKING:
+    from vanna import Agent
+    from .report import EvaluationReport, ComparisonReport
+
+
+class EvaluationRunner:
+    """Run evaluations with parallel execution support.
+
+    The primary use case is comparing multiple agent variants (e.g., different LLMs)
+    on the same set of test cases. The runner executes test cases in parallel with
+    configurable concurrency to handle I/O-bound LLM operations efficiently.
+
+    Example:
+        >>> runner = EvaluationRunner(
+        ...     evaluators=[TrajectoryEvaluator(), OutputEvaluator()],
+        ...     max_concurrency=20
+        ... )
+        >>> comparison = await runner.compare_agents(
+        ...     agent_variants=[claude_variant, gpt_variant],
+        ...     test_cases=dataset.test_cases
+        ... )
+    """
+
+    def __init__(
+        self,
+        evaluators: List[Evaluator],
+        max_concurrency: int = 10,
+        observability_provider: Optional[ObservabilityProvider] = None,
+    ):
+        """Initialize the evaluation runner.
+
+        Args:
+            evaluators: List of evaluators to apply to each test case
+            max_concurrency: Maximum number of concurrent test case executions
+            observability_provider: Optional observability for tracking eval runs
+        """
+        self.evaluators = evaluators
+        self.max_concurrency = max_concurrency
+        self.observability = observability_provider
+        self._semaphore = asyncio.Semaphore(max_concurrency)
+
+    async def run_evaluation(
+        self,
+        agent: "Agent",
+        test_cases: List[TestCase],
+    ) -> "EvaluationReport":
+        """Run evaluation on a single agent.
+
+        Args:
+            agent: The agent to evaluate
+            test_cases: List of test cases to run
+
+        Returns:
+            EvaluationReport with results for all test cases
+        """
+        from .report import EvaluationReport
+
+        results = await self._run_test_cases_parallel(agent, test_cases)
+        return EvaluationReport(
+            agent_name="agent",
+            results=results,
+            evaluators=self.evaluators,
+            timestamp=datetime.now(),
+        )
+
+    async def compare_agents(
+        self,
+        agent_variants: List[AgentVariant],
+        test_cases: List[TestCase],
+    ) -> "ComparisonReport":
+        """Compare multiple agent variants on same test cases.
+
+        This is the PRIMARY use case for LLM comparison. Runs all variants
+        in parallel for maximum efficiency with I/O-bound LLM calls.
+
+        Args:
+            agent_variants: List of agent variants to compare
+            test_cases: Test cases to run on each variant
+
+        Returns:
+            ComparisonReport with results for all variants
+        """
+        from .report import ComparisonReport
+
+        # Create span for overall comparison
+        if self.observability:
+            span = await self.observability.create_span(
+                "agent_comparison",
+                attributes={
+                    "num_variants": len(agent_variants),
+                    "num_test_cases": len(test_cases),
+                },
+            )
+
+        # Run all variants in parallel
+        tasks = [
+            self._run_agent_variant(variant, test_cases) for variant in agent_variants
+        ]
+
+        variant_reports = await asyncio.gather(*tasks)
+
+        if self.observability:
+            await self.observability.end_span(span)
+
+        return ComparisonReport(
+            variants=agent_variants,
+            reports=dict(zip([v.name for v in agent_variants], variant_reports)),
+            test_cases=test_cases,
+            timestamp=datetime.now(),
+        )
+
+    async def compare_agents_streaming(
+        self,
+        agent_variants: List[AgentVariant],
+        test_cases: List[TestCase],
+    ) -> AsyncGenerator[tuple[str, TestCaseResult, int, int], None]:
+        """Stream comparison results as they complete.
+
+        Useful for long-running evaluations where you want to see
+        progress updates in real-time (e.g., for UI display).
+
+        Args:
+            agent_variants: Agent variants to compare
+            test_cases: Test cases to run
+
+        Yields:
+            Tuples of (variant_name, result, completed_count, total_count)
+        """
+        queue: asyncio.Queue[tuple[str, TestCaseResult]] = asyncio.Queue()
+
+        async def worker(variant: AgentVariant) -> None:
+            """Worker that runs test cases for one variant."""
+            results = await self._run_test_cases_parallel(variant.agent, test_cases)
+            for result in results:
+                await queue.put((variant.name, result))
+
+        # Start all workers
+        workers = [asyncio.create_task(worker(v)) for v in agent_variants]
+
+        # Yield results as they arrive
+        completed = 0
+        total = len(agent_variants) * len(test_cases)
+
+        while completed < total:
+            variant_name, result = await queue.get()
+            completed += 1
+            yield variant_name, result, completed, total
+
+        # Wait for all workers to complete
+        await asyncio.gather(*workers)
+
+    async def _run_agent_variant(
+        self,
+        variant: AgentVariant,
+        test_cases: List[TestCase],
+    ) -> "EvaluationReport":
+        """Run a single agent variant on all test cases.
+
+        Args:
+            variant: The agent variant to evaluate
+            test_cases: Test cases to run
+
+        Returns:
+            EvaluationReport for this variant
+        """
+        from .report import EvaluationReport
+
+        if self.observability:
+            span = await self.observability.create_span(
+                f"variant_{variant.name}",
+                attributes={
+                    "variant": variant.name,
+                    "num_test_cases": len(test_cases),
+                    **variant.metadata,
+                },
+            )
+
+        results = await self._run_test_cases_parallel(variant.agent, test_cases)
+
+        if self.observability:
+            await self.observability.end_span(span)
+
+        return EvaluationReport(
+            agent_name=variant.name,
+            results=results,
+            evaluators=self.evaluators,
+            metadata=variant.metadata,
+            timestamp=datetime.now(),
+        )
+
+    async def _run_test_cases_parallel(
+        self,
+        agent: "Agent",
+        test_cases: List[TestCase],
+    ) -> List[TestCaseResult]:
+        """Run test cases in parallel with concurrency limit.
+
+        Args:
+            agent: The agent to run test cases on
+            test_cases: Test cases to execute
+
+        Returns:
+            List of TestCaseResult, one per test case
+        """
+        tasks = [
+            self._run_single_test_case(agent, test_case) for test_case in test_cases
+        ]
+
+        return await asyncio.gather(*tasks)
+
+    async def _run_single_test_case(
+        self,
+        agent: "Agent",
+        test_case: TestCase,
+    ) -> TestCaseResult:
+        """Run a single test case with semaphore to limit concurrency.
+
+        Args:
+            agent: The agent to execute
+            test_case: The test case to run
+
+        Returns:
+            TestCaseResult with agent execution and evaluations
+        """
+        async with self._semaphore:
+            # Execute agent
+            start_time = asyncio.get_event_loop().time()
+            agent_result = await self._execute_agent(agent, test_case)
+            execution_time = asyncio.get_event_loop().time() - start_time
+
+            # Run evaluators
+            eval_results = []
+            for evaluator in self.evaluators:
+                eval_result = await evaluator.evaluate(test_case, agent_result)
+                eval_results.append(eval_result)
+
+            return TestCaseResult(
+                test_case=test_case,
+                agent_result=agent_result,
+                evaluations=eval_results,
+                execution_time_ms=execution_time * 1000,
+            )
+
+    async def _execute_agent(
+        self,
+        agent: "Agent",
+        test_case: TestCase,
+    ) -> AgentResult:
+        """Execute agent and capture full trajectory.
+
+        Args:
+            agent: The agent to execute
+            test_case: The test case to run
+
+        Returns:
+            AgentResult with all captured data
+        """
+        components: List[UiComponent] = []
+        tool_calls: List[Dict[str, Any]] = []
+        error: Optional[str] = None
+
+        try:
+            # Create request context with user info from test case
+            # This allows the agent's UserResolver to resolve the correct user
+            request_context = RequestContext(
+                cookies={"user_id": test_case.user.id},
+                headers={},
+                metadata={"test_case_user": test_case.user},
+            )
+
+            async for component in agent.send_message(
+                request_context=request_context,
+                message=test_case.message,
+                conversation_id=test_case.conversation_id,
+            ):
+                components.append(component)
+
+        except Exception as e:
+            error = str(e)
+
+        # TODO: Extract tool calls and LLM requests from observability
+        # For now, these will be empty unless we hook into observability
+
+        return AgentResult(
+            test_case_id=test_case.id,
+            components=components,
+            tool_calls=tool_calls,
+            llm_requests=[],
+            error=error,
+        )

vanna/core/filter/__init__.py

@@ -0,0 +1,10 @@
+"""
+Conversation filtering system for managing conversation history.
+
+This module provides interfaces for filtering and transforming conversation
+history before it's sent to the LLM.
+"""
+
+from .base import ConversationFilter
+
+__all__ = ["ConversationFilter"]

vanna/core/filter/base.py

@@ -0,0 +1,67 @@
+"""
+Base conversation filter interface.
+
+Conversation filters allow you to transform conversation history before
+it's sent to the LLM for processing.
+"""
+
+from abc import ABC
+from typing import TYPE_CHECKING, List
+
+if TYPE_CHECKING:
+    from ..storage import Message
+
+
+class ConversationFilter(ABC):
+    """Filter for transforming conversation history.
+
+    Subclass this to create custom filters that can:
+    - Remove sensitive information
+    - Summarize long conversations
+    - Manage context window limits
+    - Deduplicate similar messages
+    - Prioritize recent or relevant messages
+
+    Example:
+        class ContextWindowFilter(ConversationFilter):
+            def __init__(self, max_tokens: int = 8000):
+                self.max_tokens = max_tokens
+
+            async def filter_messages(self, messages: List[Message]) -> List[Message]:
+                # Estimate tokens (rough approximation)
+                total_tokens = 0
+                filtered = []
+
+                # Keep system message and recent messages
+                for msg in reversed(messages):
+                    msg_tokens = len(msg.content or "") // 4
+                    if total_tokens + msg_tokens > self.max_tokens:
+                        break
+                    filtered.insert(0, msg)
+                    total_tokens += msg_tokens
+
+                return filtered
+
+        agent = AgentRunner(
+            llm_service=...,
+            conversation_filters=[
+                SensitiveDataFilter(),
+                ContextWindowFilter(max_tokens=8000)
+            ]
+        )
+    """
+
+    async def filter_messages(self, messages: List["Message"]) -> List["Message"]:
+        """Filter and transform conversation messages.
+
+        Args:
+            messages: List of conversation messages
+
+        Returns:
+            Filtered/transformed list of messages
+
+        Note:
+            Filters are applied in order, so messages passed to later
+            filters may already be modified by earlier filters.
+        """
+        return messages

vanna/core/lifecycle/__init__.py

@@ -0,0 +1,10 @@
+"""
+Lifecycle hook system for agent execution.
+
+This module provides hooks for intercepting and modifying agent behavior
+at various points in the execution lifecycle.
+"""
+
+from .base import LifecycleHook
+
+__all__ = ["LifecycleHook"]

vanna/core/lifecycle/base.py

@@ -0,0 +1,83 @@
+"""
+Base lifecycle hook interface.
+
+Lifecycle hooks allow you to intercept and customize agent behavior
+at key points in the execution flow.
+"""
+
+from abc import ABC
+from typing import TYPE_CHECKING, Any, Optional
+
+if TYPE_CHECKING:
+    from ..user.models import User
+    from ..tool import Tool
+    from ..tool.models import ToolContext, ToolResult
+
+
+class LifecycleHook(ABC):
+    """Hook into agent execution lifecycle.
+
+    Subclass this to create custom hooks that can:
+    - Modify messages before processing
+    - Add logging or telemetry
+    - Enforce quotas or rate limits
+    - Transform tool results
+    - Add custom validation
+
+    Example:
+        class LoggingHook(LifecycleHook):
+            async def before_message(self, user: User, message: str) -> Optional[str]:
+                print(f"User {user.username} sent: {message}")
+                return None  # Don't modify
+
+        agent = AgentRunner(
+            llm_service=...,
+            lifecycle_hooks=[LoggingHook(), QuotaCheckHook()]
+        )
+    """
+
+    async def before_message(self, user: "User", message: str) -> Optional[str]:
+        """Called before processing a user message.
+
+        Args:
+            user: User sending the message
+            message: Original message content
+
+        Returns:
+            Modified message string, or None to keep original
+
+        Raises:
+            AgentError: To halt message processing (e.g., quota exceeded)
+        """
+        return None
+
+    async def after_message(self, result: Any) -> None:
+        """Called after message has been fully processed.
+
+        Args:
+            result: Final result from message processing
+        """
+        pass
+
+    async def before_tool(self, tool: "Tool[Any]", context: "ToolContext") -> None:
+        """Called before tool execution.
+
+        Args:
+            tool: Tool about to be executed
+            context: Tool execution context
+
+        Raises:
+            AgentError: To prevent tool execution
+        """
+        pass
+
+    async def after_tool(self, result: "ToolResult") -> Optional["ToolResult"]:
+        """Called after tool execution.
+
+        Args:
+            result: Result from tool execution
+
+        Returns:
+            Modified ToolResult, or None to keep original
+        """
+        return None

vanna/core/llm/__init__.py

@@ -0,0 +1,16 @@
+"""
+LLM domain.
+
+This module provides the core abstractions for LLM services in the Vanna Agents framework.
+"""
+
+from .base import LlmService
+from .models import LlmMessage, LlmRequest, LlmResponse, LlmStreamChunk
+
+__all__ = [
+    "LlmService",
+    "LlmMessage",
+    "LlmRequest",
+    "LlmResponse",
+    "LlmStreamChunk",
+]

vanna/core/llm/base.py

@@ -0,0 +1,40 @@
+"""
+LLM domain interface.
+
+This module contains the abstract base class for LLM services.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any, AsyncGenerator, List
+
+from .models import LlmRequest, LlmResponse, LlmStreamChunk
+
+
+class LlmService(ABC):
+    """Service for LLM communication."""
+
+    @abstractmethod
+    async def send_request(self, request: LlmRequest) -> LlmResponse:
+        """Send a request to the LLM."""
+        pass
+
+    @abstractmethod
+    async def stream_request(
+        self, request: LlmRequest
+    ) -> AsyncGenerator[LlmStreamChunk, None]:
+        """Stream a request to the LLM.
+
+        Args:
+            request: The LLM request to stream
+
+        Yields:
+            LlmStreamChunk instances as they arrive
+        """
+        # This is an async generator method
+        raise NotImplementedError
+        yield  # pragma: no cover - makes this an async generator
+
+    @abstractmethod
+    async def validate_tools(self, tools: List[Any]) -> List[str]:
+        """Validate tool schemas and return any errors."""
+        pass

vanna/core/llm/models.py

@@ -0,0 +1,61 @@
+"""
+LLM domain models.
+
+This module contains data models for LLM communication.
+"""
+
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, Field
+
+from ..tool.models import ToolCall
+from ..user.models import User
+
+
+class LlmMessage(BaseModel):
+    """Message format for LLM communication."""
+
+    role: str = Field(description="Message role")
+    content: str = Field(description="Message content")
+    tool_calls: Optional[List[ToolCall]] = Field(default=None)
+    tool_call_id: Optional[str] = Field(default=None)
+
+
+class LlmRequest(BaseModel):
+    """Request to LLM service."""
+
+    messages: List[LlmMessage] = Field(description="Messages to send")
+    tools: Optional[List[Any]] = Field(
+        default=None, description="Available tools"
+    )  # Will be ToolSchema but avoiding circular import
+    user: User = Field(description="User making the request")
+    stream: bool = Field(default=False, description="Whether to stream response")
+    temperature: float = Field(default=0.7, ge=0.0, le=2.0)
+    max_tokens: Optional[int] = Field(default=None, gt=0)
+    system_prompt: Optional[str] = Field(
+        default=None, description="System prompt for the LLM"
+    )
+    metadata: Dict[str, Any] = Field(default_factory=dict)
+
+
+class LlmResponse(BaseModel):
+    """Response from LLM."""
+
+    content: Optional[str] = None
+    tool_calls: Optional[List[ToolCall]] = None
+    finish_reason: Optional[str] = None
+    usage: Optional[Dict[str, int]] = None
+    metadata: Dict[str, Any] = Field(default_factory=dict)
+
+    def is_tool_call(self) -> bool:
+        """Check if this response contains tool calls."""
+        return self.tool_calls is not None and len(self.tool_calls) > 0
+
+
+class LlmStreamChunk(BaseModel):
+    """Streaming chunk from LLM."""
+
+    content: Optional[str] = None
+    tool_calls: Optional[List[ToolCall]] = None
+    finish_reason: Optional[str] = None
+    metadata: Dict[str, Any] = Field(default_factory=dict)

vanna/core/middleware/__init__.py

@@ -0,0 +1,10 @@
+"""
+Middleware system for LLM request/response interception.
+
+This module provides middleware interfaces for intercepting and transforming
+LLM requests and responses.
+"""
+
+from .base import LlmMiddleware
+
+__all__ = ["LlmMiddleware"]

vanna/core/middleware/base.py

@@ -0,0 +1,69 @@
+"""
+Base LLM middleware interface.
+
+Middleware allows you to intercept and transform LLM requests and responses
+for caching, monitoring, content filtering, and more.
+"""
+
+from abc import ABC
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ..llm import LlmRequest, LlmResponse
+
+
+class LlmMiddleware(ABC):
+    """Middleware for intercepting LLM requests and responses.
+
+    Subclass this to create custom middleware that can:
+    - Cache LLM responses
+    - Log requests/responses
+    - Filter or modify content
+    - Track costs and usage
+    - Implement fallback strategies
+
+    Example:
+        class CachingMiddleware(LlmMiddleware):
+            def __init__(self):
+                self.cache = {}
+
+            async def before_llm_request(self, request: LlmRequest) -> LlmRequest:
+                # Could check cache here
+                return request
+
+            async def after_llm_response(self, request: LlmRequest, response: LlmResponse) -> LlmResponse:
+                # Cache the response
+                cache_key = self._compute_key(request)
+                self.cache[cache_key] = response
+                return response
+
+        agent = AgentRunner(
+            llm_service=...,
+            llm_middlewares=[CachingMiddleware(), LoggingMiddleware()]
+        )
+    """
+
+    async def before_llm_request(self, request: "LlmRequest") -> "LlmRequest":
+        """Called before sending request to LLM.
+
+        Args:
+            request: The LLM request about to be sent
+
+        Returns:
+            Modified request, or original if no changes
+        """
+        return request
+
+    async def after_llm_response(
+        self, request: "LlmRequest", response: "LlmResponse"
+    ) -> "LlmResponse":
+        """Called after receiving response from LLM.
+
+        Args:
+            request: The original request
+            response: The LLM response
+
+        Returns:
+            Modified response, or original if no changes
+        """
+        return response

vanna/core/observability/__init__.py

@@ -0,0 +1,11 @@
+"""
+Observability system for telemetry and monitoring.
+
+This module provides interfaces for collecting metrics, traces, and
+monitoring agent behavior.
+"""
+
+from .base import ObservabilityProvider
+from .models import Span, Metric
+
+__all__ = ["ObservabilityProvider", "Span", "Metric"]