vanna 0.7.9__py3-none-any.whl → 2.0.0rc1__py3-none-any.whl

vanna/core/evaluation/evaluators.py

@@ -0,0 +1,376 @@
+"""
+Built-in evaluators for common evaluation tasks.
+
+This module provides ready-to-use evaluators for:
+- Trajectory evaluation (tools called, order, efficiency)
+- Output evaluation (content matching, quality)
+- LLM-as-judge evaluation (custom criteria)
+- Efficiency evaluation (time, tokens, cost)
+"""
+
+from typing import Dict, Any, Optional
+from datetime import datetime
+
+from .base import Evaluator, TestCase, AgentResult, EvaluationResult
+from vanna.core import LlmService
+
+
+class TrajectoryEvaluator(Evaluator):
+    """Evaluate the path the agent took (tools called, order, etc).
+
+    Checks if the agent called the expected tools and didn't call
+    unexpected ones. Useful for verifying agent reasoning and planning.
+    """
+
+    @property
+    def name(self) -> str:
+        return "trajectory"
+
+    async def evaluate(
+        self, test_case: TestCase, agent_result: AgentResult
+    ) -> EvaluationResult:
+        """Evaluate tool call trajectory."""
+        if agent_result.error:
+            return EvaluationResult(
+                test_case_id=test_case.id,
+                evaluator_name=self.name,
+                passed=False,
+                score=0.0,
+                reasoning=f"Agent execution failed: {agent_result.error}",
+            )
+
+        expected = test_case.expected_outcome
+        if not expected:
+            return EvaluationResult(
+                test_case_id=test_case.id,
+                evaluator_name=self.name,
+                passed=True,
+                score=1.0,
+                reasoning="No expected outcome specified, passing by default",
+            )
+
+        tools_called = agent_result.get_tool_names_called()
+        issues = []
+        score = 1.0
+
+        # Check expected tools were called
+        if expected.tools_called:
+            for expected_tool in expected.tools_called:
+                if expected_tool not in tools_called:
+                    issues.append(f"Expected tool '{expected_tool}' was not called")
+                    score -= 0.5 / len(expected.tools_called)
+
+        # Check unexpected tools were not called
+        if expected.tools_not_called:
+            for unexpected_tool in expected.tools_not_called:
+                if unexpected_tool in tools_called:
+                    issues.append(f"Unexpected tool '{unexpected_tool}' was called")
+                    score -= 0.5 / len(expected.tools_not_called)
+
+        score = max(0.0, min(1.0, score))
+        passed = score >= 0.7  # 70% threshold
+
+        reasoning = "Trajectory evaluation: "
+        if issues:
+            reasoning += "; ".join(issues)
+        else:
+            reasoning += "All expected tools called, no unexpected tools"
+
+        return EvaluationResult(
+            test_case_id=test_case.id,
+            evaluator_name=self.name,
+            passed=passed,
+            score=score,
+            reasoning=reasoning,
+            metrics={
+                "tools_called": tools_called,
+                "num_tools_called": len(tools_called),
+                "issues": issues,
+            },
+        )
+
+
+class OutputEvaluator(Evaluator):
+    """Evaluate the final output quality.
+
+    Checks if the output contains expected content and doesn't
+    contain forbidden content. Case-insensitive substring matching.
+    """
+
+    @property
+    def name(self) -> str:
+        return "output"
+
+    async def evaluate(
+        self, test_case: TestCase, agent_result: AgentResult
+    ) -> EvaluationResult:
+        """Evaluate output content."""
+        if agent_result.error:
+            return EvaluationResult(
+                test_case_id=test_case.id,
+                evaluator_name=self.name,
+                passed=False,
+                score=0.0,
+                reasoning=f"Agent execution failed: {agent_result.error}",
+            )
+
+        expected = test_case.expected_outcome
+        if not expected:
+            return EvaluationResult(
+                test_case_id=test_case.id,
+                evaluator_name=self.name,
+                passed=True,
+                score=1.0,
+                reasoning="No expected outcome specified, passing by default",
+            )
+
+        final_answer = agent_result.get_final_answer().lower()
+        issues = []
+        score = 1.0
+
+        # Check expected content is present
+        if expected.final_answer_contains:
+            for expected_content in expected.final_answer_contains:
+                if expected_content.lower() not in final_answer:
+                    issues.append(
+                        f"Expected content '{expected_content}' not found in output"
+                    )
+                    score -= 0.5 / len(expected.final_answer_contains)
+
+        # Check forbidden content is absent
+        if expected.final_answer_not_contains:
+            for forbidden_content in expected.final_answer_not_contains:
+                if forbidden_content.lower() in final_answer:
+                    issues.append(
+                        f"Forbidden content '{forbidden_content}' found in output"
+                    )
+                    score -= 0.5 / len(expected.final_answer_not_contains)
+
+        score = max(0.0, min(1.0, score))
+        passed = score >= 0.7  # 70% threshold
+
+        reasoning = "Output evaluation: "
+        if issues:
+            reasoning += "; ".join(issues)
+        else:
+            reasoning += "All expected content present, no forbidden content"
+
+        return EvaluationResult(
+            test_case_id=test_case.id,
+            evaluator_name=self.name,
+            passed=passed,
+            score=score,
+            reasoning=reasoning,
+            metrics={
+                "output_length": len(final_answer),
+                "issues": issues,
+            },
+        )
+
+
+class LLMAsJudgeEvaluator(Evaluator):
+    """Use an LLM to judge agent performance based on custom criteria.
+
+    This evaluator uses a separate LLM to assess the quality of the
+    agent's output based on natural language criteria.
+    """
+
+    def __init__(self, judge_llm: LlmService, criteria: str):
+        """Initialize LLM-as-judge evaluator.
+
+        Args:
+            judge_llm: The LLM service to use for judging
+            criteria: Natural language description of what to evaluate
+        """
+        self.judge_llm = judge_llm
+        self.criteria = criteria
+
+    @property
+    def name(self) -> str:
+        return "llm_judge"
+
+    async def evaluate(
+        self, test_case: TestCase, agent_result: AgentResult
+    ) -> EvaluationResult:
+        """Evaluate using LLM as judge."""
+        if agent_result.error:
+            return EvaluationResult(
+                test_case_id=test_case.id,
+                evaluator_name=self.name,
+                passed=False,
+                score=0.0,
+                reasoning=f"Agent execution failed: {agent_result.error}",
+            )
+
+        final_answer = agent_result.get_final_answer()
+
+        # Build prompt for judge
+        judge_prompt = f"""You are evaluating an AI agent's response to a user query.
+
+User Query: {test_case.message}
+
+Agent's Response:
+{final_answer}
+
+Evaluation Criteria:
+{self.criteria}
+
+Please evaluate the response and provide:
+1. A score from 0.0 to 1.0 (where 1.0 is perfect)
+2. Whether it passes (score >= 0.7)
+3. Brief reasoning for your evaluation
+
+Respond in this format:
+SCORE: <number>
+PASSED: <yes/no>
+REASONING: <your explanation>
+"""
+
+        try:
+            # Call judge LLM
+            from vanna.core.llm import LlmRequest, LlmMessage
+
+            request = LlmRequest(
+                user=test_case.user,
+                messages=[LlmMessage(role="user", content=judge_prompt)],
+                temperature=0.0,  # Deterministic judging
+            )
+
+            response = await self.judge_llm.send_request(request)
+            judgment = response.content or ""
+
+            # Parse response
+            score = self._parse_score(judgment)
+            passed = self._parse_passed(judgment)
+            reasoning = self._parse_reasoning(judgment)
+
+            return EvaluationResult(
+                test_case_id=test_case.id,
+                evaluator_name=self.name,
+                passed=passed,
+                score=score,
+                reasoning=reasoning,
+                metrics={"judge_response": judgment},
+            )
+
+        except Exception as e:
+            return EvaluationResult(
+                test_case_id=test_case.id,
+                evaluator_name=self.name,
+                passed=False,
+                score=0.0,
+                reasoning=f"LLM judge evaluation failed: {str(e)}",
+            )
+
+    def _parse_score(self, judgment: str) -> float:
+        """Parse score from judge response."""
+        try:
+            for line in judgment.split("\n"):
+                if line.startswith("SCORE:"):
+                    score_str = line.replace("SCORE:", "").strip()
+                    return float(score_str)
+        except Exception:
+            pass
+        return 0.5  # Default if parsing fails
+
+    def _parse_passed(self, judgment: str) -> bool:
+        """Parse pass/fail from judge response."""
+        for line in judgment.split("\n"):
+            if line.startswith("PASSED:"):
+                passed_str = line.replace("PASSED:", "").strip().lower()
+                return passed_str in ["yes", "true", "pass"]
+        return False
+
+    def _parse_reasoning(self, judgment: str) -> str:
+        """Parse reasoning from judge response."""
+        for line in judgment.split("\n"):
+            if line.startswith("REASONING:"):
+                return line.replace("REASONING:", "").strip()
+        return judgment  # Return full judgment if no reasoning line found
+
+
+class EfficiencyEvaluator(Evaluator):
+    """Evaluate resource usage (time, tokens, cost).
+
+    Checks if the agent completed within acceptable resource limits.
+    """
+
+    def __init__(
+        self,
+        max_execution_time_ms: Optional[float] = None,
+        max_tokens: Optional[int] = None,
+        max_cost_usd: Optional[float] = None,
+    ):
+        """Initialize efficiency evaluator.
+
+        Args:
+            max_execution_time_ms: Maximum allowed execution time in milliseconds
+            max_tokens: Maximum allowed token usage
+            max_cost_usd: Maximum allowed cost in USD
+        """
+        self.max_execution_time_ms = max_execution_time_ms
+        self.max_tokens = max_tokens
+        self.max_cost_usd = max_cost_usd
+
+    @property
+    def name(self) -> str:
+        return "efficiency"
+
+    async def evaluate(
+        self, test_case: TestCase, agent_result: AgentResult
+    ) -> EvaluationResult:
+        """Evaluate resource efficiency."""
+        issues = []
+        score = 1.0
+
+        # Check execution time
+        if self.max_execution_time_ms:
+            if agent_result.execution_time_ms > self.max_execution_time_ms:
+                issues.append(
+                    f"Execution time {agent_result.execution_time_ms:.0f}ms "
+                    f"exceeded limit {self.max_execution_time_ms:.0f}ms"
+                )
+                score -= 0.33
+
+        # Check token usage
+        if self.max_tokens:
+            if agent_result.total_tokens > self.max_tokens:
+                issues.append(
+                    f"Token usage {agent_result.total_tokens} exceeded limit {self.max_tokens}"
+                )
+                score -= 0.33
+
+        # Check cost (would need cost calculation from metadata)
+        # For now, skip cost evaluation
+
+        # Check from expected outcome if specified
+        expected = test_case.expected_outcome
+        if expected and expected.max_execution_time_ms:
+            if agent_result.execution_time_ms > expected.max_execution_time_ms:
+                issues.append(
+                    f"Execution time {agent_result.execution_time_ms:.0f}ms "
+                    f"exceeded test case limit {expected.max_execution_time_ms:.0f}ms"
+                )
+                score -= 0.34
+
+        score = max(0.0, min(1.0, score))
+        passed = score >= 0.7
+
+        reasoning = "Efficiency evaluation: "
+        if issues:
+            reasoning += "; ".join(issues)
+        else:
+            reasoning += "Within resource limits"
+
+        return EvaluationResult(
+            test_case_id=test_case.id,
+            evaluator_name=self.name,
+            passed=passed,
+            score=score,
+            reasoning=reasoning,
+            metrics={
+                "execution_time_ms": agent_result.execution_time_ms,
+                "total_tokens": agent_result.total_tokens,
+                "issues": issues,
+            },
+        )

vanna/core/evaluation/report.py

@@ -0,0 +1,289 @@
+"""
+Evaluation reporting with HTML, CSV, and console output.
+
+This module provides classes for generating evaluation reports,
+including comparison reports for evaluating multiple agent variants.
+"""
+
+import csv
+from typing import List, Dict, Optional, Any
+from dataclasses import dataclass, field
+from datetime import datetime
+
+from .base import TestCaseResult, AgentVariant, Evaluator, TestCase
+
+
+@dataclass
+class EvaluationReport:
+    """Report for a single agent's evaluation results.
+
+    Attributes:
+        agent_name: Name of the agent evaluated
+        results: List of results for each test case
+        evaluators: List of evaluators used
+        metadata: Additional metadata about the agent/run
+        timestamp: When the evaluation was run
+    """
+
+    agent_name: str
+    results: List[TestCaseResult]
+    evaluators: List[Evaluator]
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    timestamp: datetime = field(default_factory=datetime.now)
+
+    def pass_rate(self) -> float:
+        """Calculate overall pass rate (0.0 to 1.0)."""
+        if not self.results:
+            return 0.0
+        passed = sum(1 for r in self.results if r.overall_passed())
+        return passed / len(self.results)
+
+    def average_score(self) -> float:
+        """Calculate average score across all test cases."""
+        if not self.results:
+            return 0.0
+        return sum(r.overall_score() for r in self.results) / len(self.results)
+
+    def average_time(self) -> float:
+        """Calculate average execution time in milliseconds."""
+        if not self.results:
+            return 0.0
+        return sum(r.execution_time_ms for r in self.results) / len(self.results)
+
+    def total_tokens(self) -> int:
+        """Calculate total tokens used across all test cases."""
+        return sum(r.agent_result.total_tokens for r in self.results)
+
+    def get_failures(self) -> List[TestCaseResult]:
+        """Get all failed test cases."""
+        return [r for r in self.results if not r.overall_passed()]
+
+    def print_summary(self) -> None:
+        """Print summary to console."""
+        print(f"\n{'=' * 80}")
+        print(f"EVALUATION REPORT: {self.agent_name}")
+        print(f"{'=' * 80}")
+        print(f"Timestamp: {self.timestamp.isoformat()}")
+        print(f"Test Cases: {len(self.results)}")
+        print(f"Pass Rate: {self.pass_rate():.1%}")
+        print(f"Average Score: {self.average_score():.2f}")
+        print(f"Average Time: {self.average_time():.0f}ms")
+        print(f"Total Tokens: {self.total_tokens()}")
+        print(f"{'=' * 80}\n")
+
+        failures = self.get_failures()
+        if failures:
+            print(f"FAILURES ({len(failures)}):")
+            for result in failures:
+                print(f"\n  Test Case: {result.test_case.id}")
+                print(f"  Message: {result.test_case.message}")
+                print(f"  Score: {result.overall_score():.2f}")
+                for eval_result in result.evaluations:
+                    if not eval_result.passed:
+                        print(
+                            f"    [{eval_result.evaluator_name}] {eval_result.reasoning}"
+                        )
+
+
+@dataclass
+class ComparisonReport:
+    """Report comparing multiple agent variants.
+
+    This is the primary report type for LLM comparison use cases.
+
+    Attributes:
+        variants: List of agent variants compared
+        reports: Dict mapping variant name to EvaluationReport
+        test_cases: Test cases used for comparison
+        timestamp: When the comparison was run
+    """
+
+    variants: List[AgentVariant]
+    reports: Dict[str, EvaluationReport]
+    test_cases: List[TestCase]
+    timestamp: datetime = field(default_factory=datetime.now)
+
+    def print_summary(self) -> None:
+        """Print comparison summary to console."""
+        print("\n" + "=" * 80)
+        print("AGENT COMPARISON SUMMARY")
+        print("=" * 80)
+        print(f"Timestamp: {self.timestamp.isoformat()}")
+        print(f"Variants: {len(self.variants)}")
+        print(f"Test Cases: {len(self.test_cases)}")
+
+        # Table of results
+        print(
+            f"\n{'Agent':<25} {'Pass Rate':<12} {'Avg Score':<12} {'Avg Time':<12} {'Tokens':<12}"
+        )
+        print("-" * 80)
+
+        for variant_name, report in self.reports.items():
+            print(
+                f"{variant_name:<25} "
+                f"{report.pass_rate():<12.1%} "
+                f"{report.average_score():<12.2f} "
+                f"{report.average_time():<12.0f} "
+                f"{report.total_tokens():<12,}"
+            )
+
+        print("=" * 80 + "\n")
+
+    def get_best_variant(self, metric: str = "score") -> str:
+        """Get the best performing variant by metric.
+
+        Args:
+            metric: Metric to optimize ('score', 'speed', 'pass_rate')
+
+        Returns:
+            Name of the best variant
+        """
+        if metric == "score":
+            return max(self.reports.items(), key=lambda x: x[1].average_score())[0]
+        elif metric == "speed":
+            return min(self.reports.items(), key=lambda x: x[1].average_time())[0]
+        elif metric == "pass_rate":
+            return max(self.reports.items(), key=lambda x: x[1].pass_rate())[0]
+        else:
+            raise ValueError(f"Unknown metric: {metric}")
+
+    def save_csv(self, path: str) -> None:
+        """Save detailed CSV for further analysis.
+
+        Each row represents one test case × one variant combination.
+        """
+        with open(path, "w", newline="") as f:
+            writer = csv.writer(f)
+
+            # Header
+            writer.writerow(
+                [
+                    "variant",
+                    "test_case_id",
+                    "test_message",
+                    "passed",
+                    "score",
+                    "execution_time_ms",
+                    "tokens",
+                    "error",
+                    "evaluator_scores",
+                ]
+            )
+
+            # Data rows
+            for variant_name, report in self.reports.items():
+                for result in report.results:
+                    evaluator_scores = {
+                        e.evaluator_name: e.score for e in result.evaluations
+                    }
+
+                    writer.writerow(
+                        [
+                            variant_name,
+                            result.test_case.id,
+                            result.test_case.message[:50],  # Truncate
+                            result.overall_passed(),
+                            result.overall_score(),
+                            result.execution_time_ms,
+                            result.agent_result.total_tokens,
+                            result.agent_result.error or "",
+                            str(evaluator_scores),
+                        ]
+                    )
+
+    def save_html(self, path: str) -> None:
+        """Save interactive HTML comparison report.
+
+        Generates a rich HTML report with:
+        - Summary statistics
+        - Charts comparing variants
+        - Side-by-side test case results
+        """
+        html = self._generate_html()
+        with open(path, "w") as f:
+            f.write(html)
+
+    def _generate_html(self) -> str:
+        """Generate HTML content for report."""
+        # Build HTML report
+        html_parts = [
+            "<!DOCTYPE html>",
+            "<html>",
+            "<head>",
+            "<title>Agent Comparison Report</title>",
+            "<style>",
+            "body { font-family: Arial, sans-serif; margin: 20px; }",
+            "h1 { color: #333; }",
+            "table { border-collapse: collapse; width: 100%; margin: 20px 0; }",
+            "th, td { border: 1px solid #ddd; padding: 12px; text-align: left; }",
+            "th { background-color: #4CAF50; color: white; }",
+            "tr:nth-child(even) { background-color: #f2f2f2; }",
+            ".passed { color: green; font-weight: bold; }",
+            ".failed { color: red; font-weight: bold; }",
+            ".best { background-color: #d4edda !important; }",
+            "</style>",
+            "</head>",
+            "<body>",
+            f"<h1>Agent Comparison Report</h1>",
+            f"<p>Generated: {self.timestamp.isoformat()}</p>",
+            f"<p>Variants: {len(self.variants)} | Test Cases: {len(self.test_cases)}</p>",
+        ]
+
+        # Summary table
+        html_parts.append("<h2>Summary</h2>")
+        html_parts.append("<table>")
+        html_parts.append(
+            "<tr><th>Agent</th><th>Pass Rate</th><th>Avg Score</th><th>Avg Time (ms)</th><th>Total Tokens</th></tr>"
+        )
+
+        best_by_score = self.get_best_variant("score")
+
+        for variant_name, report in self.reports.items():
+            row_class = "best" if variant_name == best_by_score else ""
+            html_parts.append(
+                f"<tr class='{row_class}'>"
+                f"<td>{variant_name}</td>"
+                f"<td>{report.pass_rate():.1%}</td>"
+                f"<td>{report.average_score():.2f}</td>"
+                f"<td>{report.average_time():.0f}</td>"
+                f"<td>{report.total_tokens():,}</td>"
+                f"</tr>"
+            )
+
+        html_parts.append("</table>")
+
+        # Test case details
+        html_parts.append("<h2>Test Case Details</h2>")
+
+        for i, test_case in enumerate(self.test_cases):
+            html_parts.append(f"<h3>Test Case {i + 1}: {test_case.id}</h3>")
+            html_parts.append(f"<p><strong>Message:</strong> {test_case.message}</p>")
+
+            html_parts.append("<table>")
+            html_parts.append(
+                "<tr><th>Variant</th><th>Result</th><th>Score</th><th>Time (ms)</th></tr>"
+            )
+
+            for variant_name, report in self.reports.items():
+                result = next(
+                    (r for r in report.results if r.test_case.id == test_case.id), None
+                )
+                if result:
+                    passed_class = "passed" if result.overall_passed() else "failed"
+                    passed_text = "PASS" if result.overall_passed() else "FAIL"
+
+                    html_parts.append(
+                        f"<tr>"
+                        f"<td>{variant_name}</td>"
+                        f"<td class='{passed_class}'>{passed_text}</td>"
+                        f"<td>{result.overall_score():.2f}</td>"
+                        f"<td>{result.execution_time_ms:.0f}</td>"
+                        f"</tr>"
+                    )
+
+            html_parts.append("</table>")
+
+        html_parts.append("</body>")
+        html_parts.append("</html>")
+
+        return "\n".join(html_parts)