tsugite-cli 0.3.3__py3-none-any.whl

tsugite/benchmark/core.py

@@ -0,0 +1,292 @@
+"""Core benchmark framework for evaluating Tsugite agents."""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from .discovery import BenchmarkTest, TestDiscovery
+from .execution import TestExecutor
+from .metrics import BenchmarkTestResult, ModelPerformance
+
+
+@dataclass
+class BenchmarkConfig:
+    """Configuration for benchmark runs."""
+
+    models: List[str] = field(default_factory=list)
+    categories: List[str] = field(default_factory=lambda: ["basic"])
+    timeout: int = 120  # seconds
+    parallel: bool = True
+    output_dir: Path = field(default_factory=lambda: Path("benchmark_results"))
+    repeat_count: int = 1  # Number of times to run each test for averaging
+    llm_evaluator_model: str = "openai:gpt-4o-mini"  # Model to use for LLM evaluation
+
+
+@dataclass
+class BenchmarkResult:
+    """Results from a complete benchmark run."""
+
+    config: BenchmarkConfig
+    start_time: datetime
+    end_time: datetime
+    total_duration: float
+    model_performances: Dict[str, ModelPerformance]
+    test_results: Dict[str, Dict[str, BenchmarkTestResult]]  # model -> test_id -> result
+    summary: Dict[str, Any]
+    errors: List[str] = field(default_factory=list)
+
+
+class BenchmarkRunner:
+    """Main benchmark runner for evaluating models across test suites."""
+
+    benchmark_dir: Path = Path("benchmarks")
+
+    def __init__(self, config: BenchmarkConfig):
+        """Initialize benchmark runner.
+
+        Args:
+            config: Benchmark configuration
+        """
+        self.config = config
+        self.benchmark_dir = Path(type(self).benchmark_dir)
+
+        # Initialize components
+        self.discovery = TestDiscovery(self.benchmark_dir, config.timeout)
+        self.executor = TestExecutor(config.output_dir, config.llm_evaluator_model)
+
+        # Create output directory
+        self.config.output_dir.mkdir(parents=True, exist_ok=True)
+
+    def discover_tests(
+        self, categories: Optional[List[str]] = None, agent_path: Optional[Path] = None
+    ) -> List[BenchmarkTest]:
+        """Discover benchmark tests.
+
+        Args:
+            categories: Categories to search (e.g., ["basic", "tools"])
+            agent_path: Specific agent file to test
+
+        Returns:
+            List of discovered tests
+        """
+        if categories is None:
+            categories = self.config.categories
+
+        return self.discovery.discover_tests(categories, agent_path)
+
+    async def run_benchmark(
+        self,
+        models: Optional[List[str]] = None,
+        categories: Optional[List[str]] = None,
+        test_filter: Optional[str] = None,
+        agent_path: Optional[Path] = None,
+    ) -> BenchmarkResult:
+        """Run benchmark suite against specified models.
+
+        Args:
+            models: List of models to test
+            categories: Test categories to run
+            test_filter: Filter tests by name/ID substring
+            agent_path: Specific agent file to test
+
+        Returns:
+            Benchmark results
+
+        Raises:
+            ValueError: If no models or tests found
+        """
+        start_time = datetime.now()
+
+        # Validate models
+        if models is None:
+            models = self.config.models
+        if not models:
+            raise ValueError("No models specified for benchmarking")
+
+        # Discover tests
+        if agent_path:
+            tests = self.discovery.discover_tests(agent_path=agent_path)
+        else:
+            tests = self.discovery.discover_tests(categories)
+            if test_filter:
+                tests = [
+                    t
+                    for t in tests
+                    if test_filter.lower() in t.name.lower() or test_filter.lower() in t.test_id.lower()
+                ]
+
+        if not tests:
+            raise ValueError("No tests found matching criteria")
+
+        print(f"Running {len(tests)} tests across {len(models)} models...")
+
+        # Run tests for each model
+        model_performances = {}
+        test_results = {model: {} for model in models}
+        errors = []
+
+        for model_name in models:
+            print(f"\nEvaluating model: {model_name}")
+            try:
+                model_perf, model_test_results, model_errors = await self._run_model_tests(model_name, tests)
+                model_performances[model_name] = model_perf
+                test_results[model_name] = model_test_results
+                errors.extend(model_errors)
+            except Exception as e:
+                error_msg = f"Failed to evaluate model {model_name}: {e}"
+                errors.append(error_msg)
+                print(f"Error: {error_msg}")
+
+        end_time = datetime.now()
+        total_duration = (end_time - start_time).total_seconds()
+
+        # Generate summary
+        summary = self._generate_summary(model_performances, test_results)
+
+        return BenchmarkResult(
+            config=self.config,
+            start_time=start_time,
+            end_time=end_time,
+            total_duration=total_duration,
+            model_performances=model_performances,
+            test_results=test_results,
+            summary=summary,
+            errors=errors,
+        )
+
+    async def _run_model_tests(
+        self, model_name: str, tests: List[BenchmarkTest]
+    ) -> tuple[ModelPerformance, Dict[str, BenchmarkTestResult], List[str]]:
+        """Run all tests for a single model.
+
+        Args:
+            model_name: Model to test
+            tests: List of tests to run
+
+        Returns:
+            Tuple of (model_performance, test_results, errors)
+        """
+        model_test_results = {}
+        model_errors = []
+
+        # Aggregate metrics
+        total_tests = len(tests)
+        passed_tests = 0
+        total_duration = 0.0
+        total_tokens = 0
+        total_cost = 0.0
+        total_steps = 0
+
+        for test in tests:
+            print(f"  Running test: {test.test_id}")
+
+            try:
+                test_result = await self.executor.run_test(model_name, test)
+                model_test_results[test.test_id] = test_result
+
+                if test_result.error:
+                    model_errors.append(f"Test {test.test_id} for model {model_name}: {test_result.error}")
+
+                if test_result.passed:
+                    passed_tests += 1
+
+                total_duration += test_result.duration
+                total_tokens += test_result.token_usage.get("total", 0)
+                total_cost += test_result.cost
+                total_steps += test_result.steps_taken
+
+            except Exception as e:
+                error_msg = f"Test {test.test_id} for model {model_name} failed: {e}"
+                model_errors.append(error_msg)
+                print(f"    Error: {error_msg}")
+
+                # Create failed test result
+                failed_result = BenchmarkTestResult(
+                    test_id=test.test_id,
+                    model=model_name,
+                    passed=False,
+                    score=0.0,
+                    duration=0.0,
+                    output="",
+                    expected_output=test.expected_output or "",
+                    error=str(e),
+                    token_usage={},
+                    cost=0.0,
+                    metrics={},
+                )
+                model_test_results[test.test_id] = failed_result
+
+        # Calculate overall metrics
+        accuracy = passed_tests / total_tests if total_tests > 0 else 0.0
+        avg_duration = total_duration / total_tests if total_tests > 0 else 0.0
+        avg_steps = total_steps / total_tests if total_tests > 0 else 0.0
+
+        model_performance = ModelPerformance(
+            model=model_name,
+            total_tests=total_tests,
+            passed_tests=passed_tests,
+            accuracy=accuracy,
+            average_duration=avg_duration,
+            total_duration=total_duration,
+            total_tokens=total_tokens,
+            total_cost=total_cost,
+            scores_by_category={},  # Will be calculated in summary
+            average_steps=avg_steps,
+        )
+
+        return model_performance, model_test_results, model_errors
+
+    def _generate_summary(
+        self,
+        model_performances: Dict[str, ModelPerformance],
+        test_results: Dict[str, Dict[str, BenchmarkTestResult]],
+    ) -> Dict[str, Any]:
+        """Generate summary statistics from benchmark results.
+
+        Args:
+            model_performances: Performance data per model
+            test_results: Detailed test results per model
+
+        Returns:
+            Summary dictionary
+        """
+        summary = {
+            "total_models": len(model_performances),
+            "model_rankings": [],
+            "category_performance": {},
+            "best_model": None,
+            "worst_model": None,
+            "average_accuracy": 0.0,
+            "total_tests": 0,
+        }
+
+        if not model_performances:
+            return summary
+
+        # Calculate model rankings by accuracy
+        ranked_models = sorted(model_performances.items(), key=lambda x: x[1].accuracy, reverse=True)
+
+        summary["model_rankings"] = [
+            {
+                "model": model,
+                "accuracy": perf.accuracy,
+                "avg_duration": perf.average_duration,
+                "total_cost": perf.total_cost,
+            }
+            for model, perf in ranked_models
+        ]
+
+        summary["best_model"] = ranked_models[0][0] if ranked_models else None
+        summary["worst_model"] = ranked_models[-1][0] if ranked_models else None
+
+        # Calculate average accuracy
+        total_accuracy = sum(perf.accuracy for perf in model_performances.values())
+        summary["average_accuracy"] = total_accuracy / len(model_performances)
+
+        # Get total tests from first model
+        if model_performances:
+            first_model = next(iter(model_performances.values()))
+            summary["total_tests"] = first_model.total_tests
+
+        return summary

tsugite/benchmark/discovery.py

@@ -0,0 +1,377 @@
+"""Test discovery and parsing for benchmark framework."""
+
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from ..utils import parse_yaml_frontmatter
+from .utils import (
+    extract_block,
+    extract_inline_field,
+    extract_prompt_from_markdown,
+    parse_bullet_list,
+    parse_key_value_block,
+)
+
+
+@dataclass
+class TestCase:
+    """Individual test case within a benchmark test."""
+
+    name: str
+    prompt: str
+    expected_output: Optional[str] = None
+    evaluation: Dict[str, Any] = field(default_factory=dict)
+    weight: float = 1.0
+    requires_plan: bool = False
+    expected_plan_elements: List[str] = field(default_factory=list)
+    plan_evaluation: Dict[str, Any] = field(default_factory=dict)
+    # LLM evaluation fields
+    use_llm_evaluation: bool = False
+    llm_evaluation_criteria: str = ""
+    llm_evaluation_rubric: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class BenchmarkTest:
+    """Test specification for an agent benchmark."""
+
+    name: str
+    agent_path: Path
+    test_id: str
+    category: str
+    description: str = ""
+    expected_output: Optional[str] = None
+    expected_type: str = "string"
+    timeout: int = 60
+    weight: float = 1.0
+    requires_tools: List[str] = field(default_factory=list)
+    test_cases: List[TestCase] = field(default_factory=list)
+    evaluation_criteria: Dict[str, Any] = field(default_factory=dict)
+    test_path: Optional[Path] = None
+    # LLM evaluation fields
+    use_llm_evaluation: bool = False
+    llm_evaluation_criteria: str = ""
+    llm_evaluation_rubric: Dict[str, Any] = field(default_factory=dict)
+
+
+class TestDiscovery:
+    """Handles discovery and parsing of benchmark tests."""
+
+    def __init__(self, benchmark_dir: Path, default_timeout: int = 120):
+        """Initialize test discovery.
+
+        Args:
+            benchmark_dir: Root directory containing benchmark tests
+            default_timeout: Default timeout for tests in seconds
+        """
+        self.benchmark_dir = Path(benchmark_dir)
+        self.default_timeout = default_timeout
+        self.test_cache: Dict[str, List[BenchmarkTest]] = {}
+
+    def discover_tests(
+        self, categories: Optional[List[str]] = None, agent_path: Optional[Path] = None
+    ) -> List[BenchmarkTest]:
+        """Discover agent + test.md pairs in specified categories or for a specific agent.
+
+        Args:
+            categories: List of category names to search (e.g., ["basic", "tools"])
+            agent_path: Specific agent file to test (bypasses category search)
+
+        Returns:
+            List of discovered benchmark tests
+        """
+        # Test a specific agent
+        if agent_path:
+            agent_path = Path(agent_path)
+            category = agent_path.parent.name if agent_path.parent.parent == self.benchmark_dir else "custom"
+            test_file = agent_path.with_suffix(".test.md")
+
+            if test_file.exists():
+                return [self._parse_agent_test_pair(agent_path, test_file, category)]
+            return [self._parse_benchmark_test(agent_path, category)]
+
+        # Discover tests from categories
+        if categories is None:
+            categories = ["basic"]
+
+        tests = []
+        for category in categories:
+            # Check cache first
+            if category in self.test_cache:
+                tests.extend(self.test_cache[category])
+                continue
+
+            # Discover from filesystem
+            category_dir = self.benchmark_dir / category
+            if not category_dir.exists():
+                print(f"Warning: Category directory not found: {category_dir}")
+                continue
+
+            category_tests = self._discover_category_tests(category)
+            self.test_cache[category] = category_tests
+            tests.extend(category_tests)
+
+        return tests
+
+    def _discover_category_tests(self, category: str) -> List[BenchmarkTest]:
+        """Discover all tests in a category directory.
+
+        Args:
+            category: Category name
+
+        Returns:
+            List of tests in this category
+        """
+        category_dir = self.benchmark_dir / category
+        category_tests = []
+
+        for agent_file in category_dir.glob("*.md"):
+            # Skip test definition files and documentation
+            if agent_file.name.endswith(".test.md") or agent_file.name == "README.md":
+                continue
+
+            test_file = agent_file.with_suffix(".test.md")
+
+            try:
+                if test_file.exists():
+                    test = self._parse_agent_test_pair(agent_file, test_file, category)
+                else:
+                    test = self._parse_benchmark_test(agent_file, category)
+                category_tests.append(test)
+            except Exception as e:
+                print(f"Error parsing benchmark test {agent_file}: {e}")
+
+        return category_tests
+
+    def _parse_agent_test_pair(self, agent_path: Path, test_path: Path, category: str) -> BenchmarkTest:
+        """Parse an agent + test.md pair into a BenchmarkTest.
+
+        Args:
+            agent_path: Path to agent definition file
+            test_path: Path to test definition file
+            category: Test category
+
+        Returns:
+            Parsed benchmark test
+
+        Raises:
+            ValueError: If parsing fails
+        """
+        try:
+            test_metadata, markdown_content = parse_yaml_frontmatter(test_path.read_text(), "Test file")
+
+            common = self._extract_common_test_fields(
+                test_metadata,
+                fallback_id=agent_path.stem,
+                fallback_description=f"Test for {agent_path.stem}",
+            )
+
+            test_cases = self._parse_test_cases(markdown_content)
+
+            # Validate agent file exists
+            if not agent_path.exists():
+                raise ValueError(f"Agent file not found: {agent_path}")
+
+            # Use first test case's expected output if no global output specified
+            expected_output = common["expected_output"]
+            if expected_output is None and test_cases:
+                expected_output = test_cases[0].expected_output
+
+            return BenchmarkTest(
+                name=agent_path.stem,
+                agent_path=agent_path,
+                test_id=common["test_id"],
+                category=category,
+                description=common["description"],
+                expected_output=expected_output,
+                expected_type=common["expected_type"],
+                timeout=common["timeout"],
+                weight=common["weight"],
+                requires_tools=common["requires_tools"],
+                test_cases=test_cases,
+                evaluation_criteria=common["evaluation_criteria"],
+                test_path=test_path,
+                use_llm_evaluation=test_metadata.get("use_llm_evaluation", False),
+                llm_evaluation_criteria=test_metadata.get("llm_evaluation_criteria", ""),
+                llm_evaluation_rubric=test_metadata.get("llm_evaluation_rubric", {}),
+            )
+        except Exception as e:
+            raise ValueError(f"Failed to parse agent test pair {agent_path}/{test_path}: {e}")
+
+    def _parse_benchmark_test(self, agent_path: Path, category: str) -> BenchmarkTest:
+        """Parse a single benchmark markdown file with embedded expectations.
+
+        Args:
+            agent_path: Path to agent/benchmark file
+            category: Test category
+
+        Returns:
+            Parsed benchmark test
+
+        Raises:
+            ValueError: If parsing fails
+        """
+        try:
+            metadata, markdown_content = parse_yaml_frontmatter(agent_path.read_text(), "Benchmark file")
+
+            common = self._extract_common_test_fields(
+                metadata,
+                fallback_id=agent_path.stem,
+                fallback_description=metadata.get("name", ""),
+            )
+
+            prompt = metadata.get("prompt") or extract_prompt_from_markdown(markdown_content)
+
+            test_case = TestCase(
+                name=f"{common['test_id']}_default",
+                prompt=prompt,
+                expected_output=common["expected_output"],
+                evaluation=metadata.get("evaluation", {}),
+                weight=common["weight"],
+                requires_plan=metadata.get("requires_plan", False),
+                expected_plan_elements=metadata.get("expected_plan_elements", []),
+                plan_evaluation=metadata.get("plan_evaluation", {}),
+                use_llm_evaluation=metadata.get("use_llm_evaluation", False),
+                llm_evaluation_criteria=metadata.get("llm_evaluation_criteria", ""),
+                llm_evaluation_rubric=metadata.get("llm_evaluation_rubric", {}),
+            )
+
+            return BenchmarkTest(
+                name=metadata.get("name", agent_path.stem),
+                agent_path=agent_path,
+                test_id=common["test_id"],
+                category=category,
+                description=common["description"],
+                expected_output=common["expected_output"],
+                expected_type=common["expected_type"],
+                timeout=common["timeout"],
+                weight=common["weight"],
+                requires_tools=common["requires_tools"],
+                test_cases=[test_case],
+                evaluation_criteria=common["evaluation_criteria"],
+                test_path=None,
+                use_llm_evaluation=metadata.get("use_llm_evaluation", False),
+                llm_evaluation_criteria=metadata.get("llm_evaluation_criteria", ""),
+                llm_evaluation_rubric=metadata.get("llm_evaluation_rubric", {}),
+            )
+        except Exception as e:
+            raise ValueError(f"Failed to parse benchmark test {agent_path}: {e}")
+
+    def _extract_common_test_fields(
+        self,
+        metadata: Dict[str, Any],
+        fallback_id: str,
+        fallback_description: str,
+    ) -> Dict[str, Any]:
+        """Extract common metadata fields from test definition.
+
+        Args:
+            metadata: Parsed YAML frontmatter
+            fallback_id: ID to use if not specified
+            fallback_description: Description to use if not specified
+
+        Returns:
+            Dictionary of common fields
+        """
+        weight_value = metadata.get("weight", 1.0)
+        try:
+            weight = float(weight_value)
+        except (TypeError, ValueError):
+            weight = 1.0
+
+        requires_tools = metadata.get("requires_tools", metadata.get("tools", []))
+
+        return {
+            "test_id": metadata.get("test_id", fallback_id),
+            "description": metadata.get("description", fallback_description),
+            "timeout": metadata.get("timeout", self.default_timeout),
+            "requires_tools": requires_tools,
+            "weight": weight,
+            "expected_output": metadata.get("expected_output"),
+            "expected_type": metadata.get("expected_type", "string"),
+            "evaluation_criteria": metadata.get("evaluation_criteria", {}),
+        }
+
+    def _parse_test_cases(self, markdown_content: str) -> List[TestCase]:
+        """Parse test cases from markdown content.
+
+        Args:
+            markdown_content: Markdown content containing test cases
+
+        Returns:
+            List of parsed test cases
+        """
+        test_cases = []
+
+        # Split content by ## headers (test cases)
+        sections = re.split(r"\n## (.+?)\n", markdown_content)
+
+        # Skip the first section (usually intro text)
+        for i in range(1, len(sections), 2):
+            if i + 1 >= len(sections):
+                break
+
+            case_name = sections[i].strip()
+            case_content = sections[i + 1].strip()
+
+            # Parse the test case content
+            test_case = self._parse_single_test_case(case_name, case_content)
+            if test_case:
+                test_cases.append(test_case)
+
+        return test_cases
+
+    def _parse_single_test_case(self, name: str, content: str) -> Optional[TestCase]:
+        """Parse a single test case from markdown content.
+
+        Args:
+            name: Test case name
+            content: Test case markdown content
+
+        Returns:
+            Parsed test case or None if parsing fails
+        """
+        # Extract prompt - required
+        prompt = extract_inline_field(content, "Prompt")
+        if not prompt:
+            return None
+
+        # Extract expected output
+        expected_output = extract_inline_field(content, "Expected Output")
+
+        # Parse evaluation criteria
+        evaluation = parse_key_value_block(extract_block(content, "Evaluation"))
+
+        # Parse planning requirements
+        requires_plan_text = extract_inline_field(content, "Requires Plan")
+        requires_plan = bool(requires_plan_text and requires_plan_text.strip().lower() in {"true", "yes", "1"})
+
+        expected_plan_elements = parse_bullet_list(extract_block(content, "Expected Plan Elements"))
+        plan_evaluation = parse_key_value_block(extract_block(content, "Plan Evaluation"))
+
+        # Parse LLM evaluation fields
+        use_llm_evaluation_text = extract_inline_field(content, "Use LLM Evaluation")
+        use_llm_evaluation = bool(
+            use_llm_evaluation_text and use_llm_evaluation_text.strip().lower() in {"true", "yes", "1"}
+        )
+
+        llm_evaluation_criteria = extract_inline_field(content, "LLM Evaluation Criteria") or ""
+
+        llm_rubric_block = extract_block(content, "LLM Evaluation Rubric")
+        llm_evaluation_rubric = parse_key_value_block(llm_rubric_block)
+
+        return TestCase(
+            name=name,
+            prompt=prompt,
+            expected_output=expected_output,
+            evaluation=evaluation,
+            weight=1.0,
+            requires_plan=requires_plan,
+            expected_plan_elements=expected_plan_elements,
+            plan_evaluation=plan_evaluation,
+            use_llm_evaluation=use_llm_evaluation,
+            llm_evaluation_criteria=llm_evaluation_criteria,
+            llm_evaluation_rubric=llm_evaluation_rubric,
+        )