tsugite-cli 0.3.3__py3-none-any.whl

tsugite/benchmark/metrics.py

@@ -0,0 +1,204 @@
+"""Metrics and data structures for benchmark results."""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
+
+@dataclass
+class BenchmarkTestResult:
+    """Result from running a single test."""
+
+    test_id: str
+    model: str
+    passed: bool
+    score: float  # 0.0 to 1.0
+    duration: float  # seconds
+    output: str
+    expected_output: str
+    category: str = "unknown"
+    error: Optional[str] = None
+    token_usage: Dict[str, int] = field(default_factory=dict)
+    cost: float = 0.0
+    steps_taken: int = 0  # Number of reasoning steps/attempts
+    metrics: Dict[str, Any] = field(default_factory=dict)
+    timestamp: datetime = field(default_factory=datetime.now)
+
+
+@dataclass
+class ModelPerformance:
+    """Aggregated performance metrics for a model."""
+
+    model: str
+    total_tests: int
+    passed_tests: int
+    accuracy: float  # passed_tests / total_tests
+    average_duration: float
+    total_duration: float
+    total_tokens: int
+    total_cost: float
+    scores_by_category: Dict[str, float] = field(default_factory=dict)
+    error_rate: float = 0.0
+    reliability_score: float = 0.0
+    average_steps: float = 0.0
+
+
+@dataclass
+class CategoryMetrics:
+    """Performance metrics for a test category."""
+
+    category: str
+    total_tests: int
+    model_scores: Dict[str, float]  # model -> average score
+    average_score: float
+    best_model: str
+    worst_model: str
+
+
+class BenchmarkMetrics:
+    """Utility class for calculating benchmark metrics."""
+
+    @staticmethod
+    def calculate_accuracy(passed: int, total: int) -> float:
+        """Calculate accuracy percentage."""
+        if total == 0:
+            return 0.0
+        return passed / total
+
+    @staticmethod
+    def calculate_efficiency_score(duration: float, baseline_duration: float) -> float:
+        """Calculate efficiency score relative to baseline (higher is better)."""
+        if baseline_duration <= 0:
+            return 1.0
+        return min(baseline_duration / duration, 1.0)
+
+    @staticmethod
+    def calculate_cost_efficiency(score: float, cost: float) -> float:
+        """Calculate cost efficiency (score per unit cost)."""
+        if cost <= 0:
+            return score
+        return score / cost
+
+    @staticmethod
+    def calculate_weighted_score(scores: Dict[str, float], weights: Dict[str, float]) -> float:
+        """Calculate weighted average of scores."""
+        total_weight = sum(weights.values())
+        if total_weight == 0:
+            return 0.0
+
+        weighted_sum = sum(scores.get(metric, 0.0) * weight for metric, weight in weights.items())
+        return weighted_sum / total_weight
+
+    @staticmethod
+    def calculate_reliability_score(test_results: List[BenchmarkTestResult]) -> float:
+        """Calculate reliability score based on error patterns."""
+        if not test_results:
+            return 0.0
+
+        total_tests = len(test_results)
+        error_count = sum(1 for result in test_results if result.error is not None)
+        timeout_count = sum(1 for result in test_results if result.error and "timeout" in result.error.lower())
+
+        # Penalize errors and timeouts more heavily
+        error_penalty = (error_count * 0.5 + timeout_count * 0.3) / total_tests
+        reliability = max(0.0, 1.0 - error_penalty)
+
+        return reliability
+
+    @staticmethod
+    def calculate_category_performance(test_results: Dict[str, BenchmarkTestResult], category: str) -> CategoryMetrics:
+        """Calculate performance metrics for a specific category."""
+        category_results = [result for result in test_results.values() if result.test_id.startswith(category)]
+
+        if not category_results:
+            return CategoryMetrics(
+                category=category,
+                total_tests=0,
+                model_scores={},
+                average_score=0.0,
+                best_model="",
+                worst_model="",
+            )
+
+        # Group by model
+        model_scores = {}
+        for result in category_results:
+            if result.model not in model_scores:
+                model_scores[result.model] = []
+            model_scores[result.model].append(result.score)
+
+        # Calculate average scores per model
+        avg_model_scores = {model: round(sum(scores) / len(scores), 4) for model, scores in model_scores.items()}
+
+        # Find best and worst models
+        if avg_model_scores:
+            best_model = max(avg_model_scores.items(), key=lambda x: x[1])[0]
+            worst_model = min(avg_model_scores.items(), key=lambda x: x[1])[0]
+            average_score = round(sum(avg_model_scores.values()) / len(avg_model_scores), 4)
+        else:
+            best_model = worst_model = ""
+            average_score = 0.0
+
+        return CategoryMetrics(
+            category=category,
+            total_tests=len(category_results),
+            model_scores=avg_model_scores,
+            average_score=average_score,
+            best_model=best_model,
+            worst_model=worst_model,
+        )
+
+    @staticmethod
+    def get_performance_tier(accuracy: float) -> str:
+        """Classify performance into tiers."""
+        if accuracy >= 0.9:
+            return "Excellent"
+        elif accuracy >= 0.75:
+            return "Good"
+        elif accuracy >= 0.6:
+            return "Fair"
+        elif accuracy >= 0.4:
+            return "Poor"
+        else:
+            return "Very Poor"
+
+    @staticmethod
+    def calculate_improvement_percentage(baseline_score: float, current_score: float) -> float:
+        """Calculate percentage improvement over baseline."""
+        if baseline_score <= 0:
+            return 0.0
+
+        improvement = ((current_score - baseline_score) / baseline_score) * 100
+        return round(improvement, 4)
+
+    @staticmethod
+    def aggregate_test_results(results: List[BenchmarkTestResult]) -> Dict[str, Any]:
+        """Aggregate multiple test results into summary statistics."""
+        if not results:
+            return {
+                "total_tests": 0,
+                "passed_tests": 0,
+                "accuracy": 0.0,
+                "average_score": 0.0,
+                "average_duration": 0.0,
+                "total_cost": 0.0,
+                "error_rate": 0.0,
+            }
+
+        total_tests = len(results)
+        passed_tests = sum(1 for r in results if r.passed)
+        total_score = sum(r.score for r in results)
+        total_duration = sum(r.duration for r in results)
+        total_cost = sum(r.cost for r in results)
+        error_count = sum(1 for r in results if r.error is not None)
+
+        return {
+            "total_tests": total_tests,
+            "passed_tests": passed_tests,
+            "accuracy": passed_tests / total_tests,
+            "average_score": total_score / total_tests,
+            "average_duration": total_duration / total_tests,
+            "total_duration": total_duration,
+            "total_cost": total_cost,
+            "error_rate": error_count / total_tests,
+        }

tsugite/benchmark/reports.py

@@ -0,0 +1,420 @@
+"""Report generation for benchmark results."""
+
+import json
+from dataclasses import asdict
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List
+
+from .config import get_performance_tier
+from .core import BenchmarkResult
+
+
+class ReportGenerator:
+    """Generate various formats of benchmark reports."""
+
+    def __init__(self, result: BenchmarkResult):
+        self.result = result
+
+    def generate_json_report(self, output_path: Path) -> None:
+        """Generate a comprehensive JSON report."""
+        report_data = {
+            "benchmark_info": {
+                "start_time": self.result.start_time.isoformat(),
+                "end_time": self.result.end_time.isoformat(),
+                "total_duration": self.result.total_duration,
+                "models_tested": list(self.result.model_performances.keys()),
+                "total_tests": len(self._get_all_test_ids()),
+            },
+            "config": asdict(self.result.config),
+            "summary": self.result.summary,
+            "model_performances": {model: asdict(perf) for model, perf in self.result.model_performances.items()},
+            "detailed_results": {
+                model: {test_id: asdict(result) for test_id, result in tests.items()}
+                for model, tests in self.result.test_results.items()
+            },
+            "errors": self.result.errors,
+            "generated_at": datetime.now().isoformat(),
+        }
+
+        with open(output_path, "w") as f:
+            json.dump(report_data, f, indent=2, default=str)
+
+    def generate_markdown_report(self, output_path: Path) -> None:
+        """Generate a markdown report for easy reading."""
+        report = []
+
+        # Header
+        report.append("# Tsugite Benchmark Report")
+        report.append(f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
+        report.append("")
+
+        # Summary
+        report.append("## Summary")
+        report.append(f"- **Duration**: {self.result.total_duration:.2f} seconds")
+        report.append(f"- **Models Tested**: {len(self.result.model_performances)}")
+        report.append(f"- **Total Tests**: {self.result.summary.get('total_tests', 0)}")
+        report.append(f"- **Average Accuracy**: {self.result.summary.get('average_accuracy', 0):.1%}")
+        report.append("")
+
+        # Model Rankings
+        if "model_rankings" in self.result.summary:
+            report.append("## Model Rankings")
+            report.append("| Rank | Model | Accuracy | Avg Duration | Total Cost |")
+            report.append("|------|-------|----------|--------------|------------|")
+
+            for i, ranking in enumerate(self.result.summary["model_rankings"], 1):
+                model = ranking["model"]
+                accuracy = f"{ranking['accuracy']:.1%}"
+                duration = f"{ranking['avg_duration']:.2f}s"
+                cost = f"${ranking['total_cost']:.4f}"
+                report.append(f"| {i} | {model} | {accuracy} | {duration} | {cost} |")
+            report.append("")
+
+        # Detailed Performance
+        report.append("## Detailed Performance")
+        for model, performance in self.result.model_performances.items():
+            report.append(f"### {model}")
+            report.append(
+                f"- **Accuracy**: {performance.accuracy:.1%} ({performance.passed_tests}/{performance.total_tests})"
+            )
+            report.append(f"- **Average Duration**: {performance.average_duration:.2f}s")
+            report.append(f"- **Average Steps**: {performance.average_steps:.1f}")
+            report.append(f"- **Total Cost**: ${performance.total_cost:.4f}")
+            report.append(f"- **Performance Tier**: {get_performance_tier(performance.accuracy)}")
+            report.append("")
+
+        # Category Breakdown
+        category_breakdown = self._calculate_category_breakdown()
+        if category_breakdown:
+            report.append("## Performance by Category")
+            for category, data in category_breakdown.items():
+                report.append(f"### {category.title()}")
+                report.append(f"- **Best Model**: {data['best_model']} ({data['best_score']:.1%})")
+                report.append(f"- **Average Score**: {data['average_score']:.1%}")
+                report.append("")
+
+        # Test Details
+        report.append("## Test Results Details")
+        for model in self.result.model_performances.keys():
+            report.append(f"### {model}")
+            report.append("| Test ID | Category | Result | Score | Duration | Steps | Cost |")
+            report.append("|---------|----------|--------|-------|----------|-------|------|")
+
+            model_tests = self.result.test_results.get(model, {})
+            for test_id, test_result in model_tests.items():
+                category = test_result.category
+                result_status = "✅ PASS" if test_result.passed else "❌ FAIL"
+                score = f"{test_result.score:.2f}"
+                duration = f"{test_result.duration:.2f}s"
+                steps = str(test_result.steps_taken)
+                cost = f"${test_result.cost:.4f}" if test_result.cost > 0 else "$0.00"
+                report.append(f"| {test_id} | {category} | {result_status} | {score} | {duration} | {steps} | {cost} |")
+            report.append("")
+
+        # Errors
+        if self.result.errors:
+            report.append("## Errors")
+            for error in self.result.errors:
+                report.append(f"- {error}")
+            report.append("")
+
+        # Write to file
+        with open(output_path, "w") as f:
+            f.write("\n".join(report))
+
+    def generate_html_report(self, output_path: Path) -> None:
+        """Generate an interactive HTML report."""
+        html_content = self._generate_html_content()
+
+        with open(output_path, "w") as f:
+            f.write(html_content)
+
+    def generate_csv_summary(self, output_path: Path) -> None:
+        """Generate CSV summary for data analysis."""
+        import csv
+
+        with open(output_path, "w", newline="") as f:
+            writer = csv.writer(f)
+
+            # Header
+            writer.writerow(
+                [
+                    "Model",
+                    "Test_ID",
+                    "Category",
+                    "Passed",
+                    "Score",
+                    "Duration",
+                    "Steps",
+                    "Token_Usage",
+                    "Cost",
+                    "Error",
+                ]
+            )
+
+            # Data rows
+            for model, tests in self.result.test_results.items():
+                for test_id, test_result in tests.items():
+                    category = test_result.category
+                    token_usage = test_result.token_usage.get("total", 0)
+
+                    writer.writerow(
+                        [
+                            model,
+                            test_id,
+                            category,
+                            test_result.passed,
+                            test_result.score,
+                            test_result.duration,
+                            test_result.steps_taken,
+                            token_usage,
+                            test_result.cost,
+                            test_result.error or "",
+                        ]
+                    )
+
+    def _get_all_test_ids(self) -> List[str]:
+        """Get all unique test IDs across all models."""
+        all_test_ids = set()
+        for tests in self.result.test_results.values():
+            all_test_ids.update(tests.keys())
+        return list(all_test_ids)
+
+    def _calculate_category_breakdown(self) -> Dict[str, Any]:
+        """Calculate performance breakdown by category."""
+        categories = {}
+
+        for model, tests in self.result.test_results.items():
+            for test_id, test_result in tests.items():
+                category = test_result.category
+
+                if category not in categories:
+                    categories[category] = {}
+
+                if model not in categories[category]:
+                    categories[category][model] = []
+
+                categories[category][model].append(test_result.score)
+
+        # Calculate averages and find best/worst
+        breakdown = {}
+        for category, model_scores in categories.items():
+            avg_scores = {model: sum(scores) / len(scores) for model, scores in model_scores.items()}
+
+            if avg_scores:
+                best_model = max(avg_scores.items(), key=lambda x: x[1])
+                average_score = sum(avg_scores.values()) / len(avg_scores)
+
+                breakdown[category] = {
+                    "best_model": best_model[0],
+                    "best_score": best_model[1],
+                    "average_score": average_score,
+                    "model_scores": avg_scores,
+                }
+
+        return breakdown
+
+    def _generate_html_content(self) -> str:
+        """Generate HTML content for the report."""
+        html = f"""
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Tsugite Benchmark Report</title>
+    <style>
+        body {{
+            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
+            line-height: 1.6;
+            margin: 0;
+            padding: 20px;
+            background-color: #f5f5f5;
+        }}
+        .container {{
+            max-width: 1200px;
+            margin: 0 auto;
+            background: white;
+            padding: 30px;
+            border-radius: 8px;
+            box-shadow: 0 2px 10px rgba(0,0,0,0.1);
+        }}
+        h1, h2, h3 {{
+            color: #333;
+        }}
+        .summary {{
+            background: #f8f9fa;
+            padding: 20px;
+            border-radius: 6px;
+            margin: 20px 0;
+        }}
+        .metric {{
+            display: inline-block;
+            margin: 10px 20px 10px 0;
+        }}
+        .metric-value {{
+            font-size: 1.5em;
+            font-weight: bold;
+            color: #007bff;
+        }}
+        .metric-label {{
+            font-size: 0.9em;
+            color: #666;
+        }}
+        table {{
+            width: 100%;
+            border-collapse: collapse;
+            margin: 20px 0;
+        }}
+        th, td {{
+            padding: 12px;
+            text-align: left;
+            border-bottom: 1px solid #ddd;
+        }}
+        th {{
+            background-color: #f8f9fa;
+            font-weight: 600;
+        }}
+        .pass {{ color: #28a745; }}
+        .fail {{ color: #dc3545; }}
+        .tier-excellent {{ color: #28a745; font-weight: bold; }}
+        .tier-good {{ color: #17a2b8; }}
+        .tier-fair {{ color: #ffc107; }}
+        .tier-poor {{ color: #fd7e14; }}
+        .tier-very-poor {{ color: #dc3545; }}
+        .progress-bar {{
+            background: #e9ecef;
+            border-radius: 10px;
+            height: 20px;
+            margin: 5px 0;
+        }}
+        .progress-fill {{
+            height: 100%;
+            border-radius: 10px;
+            transition: width 0.3s ease;
+        }}
+        .progress-excellent {{ background: #28a745; }}
+        .progress-good {{ background: #17a2b8; }}
+        .progress-fair {{ background: #ffc107; }}
+        .progress-poor {{ background: #fd7e14; }}
+        .progress-very-poor {{ background: #dc3545; }}
+    </style>
+</head>
+<body>
+    <div class="container">
+        <h1>🚀 Tsugite Benchmark Report</h1>
+        <p>Generated: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}</p>
+
+        <div class="summary">
+            <h2>📊 Summary</h2>
+            <div class="metric">
+                <div class="metric-value">{self.result.total_duration:.1f}s</div>
+                <div class="metric-label">Total Duration</div>
+            </div>
+            <div class="metric">
+                <div class="metric-value">{len(self.result.model_performances)}</div>
+                <div class="metric-label">Models Tested</div>
+            </div>
+            <div class="metric">
+                <div class="metric-value">{self.result.summary.get("total_tests", 0)}</div>
+                <div class="metric-label">Total Tests</div>
+            </div>
+            <div class="metric">
+                <div class="metric-value">{self.result.summary.get("average_accuracy", 0):.1%}</div>
+                <div class="metric-label">Average Accuracy</div>
+            </div>
+        </div>
+
+        <h2>🏆 Model Rankings</h2>
+        <table>
+            <thead>
+                <tr>
+                    <th>Rank</th>
+                    <th>Model</th>
+                    <th>Accuracy</th>
+                    <th>Performance</th>
+                    <th>Avg Duration</th>
+                    <th>Total Cost</th>
+                </tr>
+            </thead>
+            <tbody>
+        """
+
+        # Model rankings table
+        if "model_rankings" in self.result.summary:
+            for i, ranking in enumerate(self.result.summary["model_rankings"], 1):
+                accuracy = ranking["accuracy"]
+                tier_class = self._get_tier_class(accuracy)
+                progress_class = self._get_progress_class(accuracy)
+
+                html += f"""
+                <tr>
+                    <td>{i}</td>
+                    <td><strong>{ranking["model"]}</strong></td>
+                    <td>
+                        {accuracy:.1%}
+                        <div class="progress-bar">
+                            <div class="progress-fill {progress_class}" style="width: {accuracy * 100}%"></div>
+                        </div>
+                    </td>
+                    <td><span class="{tier_class}">{get_performance_tier(accuracy)}</span></td>
+                    <td>{ranking["avg_duration"]:.2f}s</td>
+                    <td>${ranking["total_cost"]:.4f}</td>
+                </tr>
+                """
+
+        html += """
+            </tbody>
+        </table>
+
+        <h2>📈 Detailed Results</h2>
+        """
+
+        # Detailed results for each model
+        for model, performance in self.result.model_performances.items():
+            tier_class = self._get_tier_class(performance.accuracy)
+            html += f"""
+            <h3>{model}</h3>
+            <p>
+                <span class="{tier_class}">
+                    {get_performance_tier(performance.accuracy)}
+                </span>
+                - {performance.accuracy:.1%} accuracy
+                ({performance.passed_tests}/{performance.total_tests} tests passed)
+            </p>
+            """
+
+        html += """
+        </div>
+    </body>
+</html>
+        """
+
+        return html
+
+    def _get_tier_class(self, accuracy: float) -> str:
+        """Get CSS class for performance tier."""
+        if accuracy >= 0.9:
+            return "tier-excellent"
+        elif accuracy >= 0.75:
+            return "tier-good"
+        elif accuracy >= 0.6:
+            return "tier-fair"
+        elif accuracy >= 0.4:
+            return "tier-poor"
+        else:
+            return "tier-very-poor"
+
+    def _get_progress_class(self, accuracy: float) -> str:
+        """Get CSS class for progress bar."""
+        if accuracy >= 0.9:
+            return "progress-excellent"
+        elif accuracy >= 0.75:
+            return "progress-good"
+        elif accuracy >= 0.6:
+            return "progress-fair"
+        elif accuracy >= 0.4:
+            return "progress-poor"
+        else:
+            return "progress-very-poor"