fableforge-bench-agent 0.1.0__py3-none-any.whl

bench_agent/__init__.py

@@ -0,0 +1,3 @@
+"""BenchAgent — HumanEval for tool use."""
+
+__version__ = "0.1.0"

bench_agent/cli.py

@@ -0,0 +1,199 @@
+"""CLI interface for BenchAgent benchmark runner."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import click
+from rich.console import Console
+from rich.table import Table
+
+from bench_agent.evaluator import ModelProvider, evaluate_model, evaluate_with_retry
+from bench_agent.leaderboard import (
+    export_leaderboard,
+    export_markdown,
+    load_leaderboard,
+    save_leaderboard,
+    update_leaderboard,
+)
+from bench_agent.models import Leaderboard, TaskCategory
+from bench_agent.runner import TaskRunner
+from bench_agent.tasks import ALL_TASKS, TASKS_BY_CATEGORY, get_task_count
+
+
+console = Console()
+
+
+@click.group()
+@click.version_option(version="0.1.0")
+def main() -> None:
+    """BenchAgent — HumanEval for tool use."""
+
+
+@main.command("list-tasks")
+@click.option("--category", "-c", type=click.Choice([c.value for c in TaskCategory]), default=None)
+@click.option("--difficulty", "-d", type=click.Choice(["easy", "medium", "hard"]), default=None)
+def list_tasks(category: str | None, difficulty: str | None) -> None:
+    """List available benchmark tasks."""
+    table = Table(title="BenchAgent Tasks")
+    table.add_column("ID", style="cyan")
+    table.add_column("Category", style="green")
+    table.add_column("Difficulty", style="yellow")
+    table.add_column("Description", style="white")
+    table.add_column("Max Turns", style="magenta")
+
+    tasks = ALL_TASKS
+    if category:
+        tasks = [t for t in tasks if t.category.value == category]
+    if difficulty:
+        tasks = [t for t in tasks if t.difficulty.value == difficulty]
+
+    for task in tasks:
+        table.add_row(
+            task.task_id,
+            task.category.value,
+            task.difficulty.value,
+            task.description[:60] + ("..." if len(task.description) > 60 else ""),
+            str(task.max_turns),
+        )
+
+    console.print(table)
+    console.print(f"\nTotal: {len(tasks)} tasks (of {get_task_count()} total)")
+
+
+@main.command("run")
+@click.option("--model", "-m", required=True, help="Model name to evaluate")
+@click.option("--category", "-c", type=click.Choice([c.value for c in TaskCategory]), default=None, help="Task category")
+@click.option("--all", "run_all", is_flag=True, help="Run all categories")
+@click.option("--num-tasks", "-n", type=int, default=None, help="Number of tasks to run")
+@click.option("--output", "-o", type=click.Path(), default=None, help="Output file for results")
+@click.option("--provider", "-p", type=click.Choice(["openai", "anthropic", "local", "huggingface"]), default="openai")
+@click.option("--timeout", type=float, default=300.0, help="Total timeout per task in seconds")
+@click.option("--retries", type=int, default=3, help="Max retries on API errors")
+def run(
+    model: str,
+    category: str | None,
+    run_all: bool,
+    num_tasks: int | None,
+    output: str | None,
+    provider: str,
+    timeout: float,
+    retries: int,
+) -> None:
+    """Run benchmark tasks against a model."""
+    categories = None
+    if run_all:
+        categories = list(TaskCategory)
+    elif category:
+        categories = [TaskCategory(category)]
+
+    console.print(f"[bold blue]Running BenchAgent for model: {model}[/bold blue]")
+    console.print(f"Provider: {provider}")
+
+    runner = TaskRunner(total_timeout=timeout)
+
+    if categories:
+        cat_names = [c.value for c in categories]
+        console.print(f"Categories: {', '.join(cat_names)}")
+    else:
+        console.print("Categories: all")
+
+    try:
+        report = evaluate_with_retry(
+            model_name=model,
+            provider=provider,
+            categories=categories,
+            num_tasks=num_tasks,
+            max_retries=retries,
+            runner=runner,
+        )
+    except Exception as e:
+        console.print(f"[bold red]Error: {e}[/bold red]")
+        sys.exit(1)
+
+    console.print(f"\n[bold green]Results for {model}[/bold green]")
+    console.print(f"Total Score: {report.total_score}")
+    console.print(f"Error Recovery Rate: {report.error_recovery_rate}")
+
+    if report.category_scores:
+        table = Table(title="Category Scores")
+        table.add_column("Category", style="cyan")
+        table.add_column("Score", style="green")
+        for cat, score in report.category_scores.items():
+            table.add_row(cat, f"{score:.1f}")
+        console.print(table)
+
+    if output:
+        output_path = Path(output)
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        output_path.write_text(report.model_dump_json(indent=2))
+        console.print(f"\nResults saved to: {output}")
+
+
+@main.command("leaderboard")
+@click.option("--update", is_flag=True, help="Update leaderboard with new results")
+@click.option("--results", type=click.Path(exists=True), help="Results file to add")
+@click.option("--path", type=click.Path(), default="leaderboard.json", help="Leaderboard file path")
+@click.option("--format", "-f", type=click.Choice(["json", "markdown"]), default="json", help="Export format")
+def leaderboard(update: bool, results: str | None, path: str, format: str) -> None:
+    """Show or update the leaderboard."""
+    lb = load_leaderboard(path)
+
+    if update and results:
+        results_path = Path(results)
+        data = json.loads(results_path.read_text())
+
+        if isinstance(data, list):
+            for entry in data:
+                lb = update_leaderboard(lb, entry.get("model", "unknown"), [])
+        elif isinstance(data, dict) and "task_id" in data:
+            entries = [data]
+        else:
+            report = ScoreReport(**data) if isinstance(data, dict) else data
+            lb.entries.append(report)
+
+        lb = update_leaderboard(lb, data.get("model", "unknown") if isinstance(data, dict) else "unknown", [])
+        save_leaderboard(lb, path)
+        console.print(f"[green]Leaderboard updated and saved to {path}[/green]")
+
+    if format == "markdown":
+        console.print(export_markdown(lb))
+    else:
+        table = Table(title="BenchAgent Leaderboard")
+        table.add_column("Rank", style="bold")
+        table.add_column("Model", style="cyan")
+        table.add_column("Score", style="green")
+        table.add_column("Recovery Rate", style="yellow")
+
+        for entry in lb.entries:
+            table.add_row(
+                str(entry.leaderboard_rank),
+                entry.model,
+                f"{entry.total_score:.1f}",
+                f"{entry.error_recovery_rate:.3f}",
+            )
+
+        console.print(table)
+        console.print(f"\nLast updated: {lb.last_updated}")
+
+
+@main.command("export")
+@click.option("--path", type=click.Path(), default="leaderboard.json", help="Leaderboard file path")
+@click.option("--format", "-f", type=click.Choice(["json", "markdown"]), default="json")
+@click.option("--output", "-o", type=click.Path(), default=None)
+def export(path: str, format: str, output: str | None) -> None:
+    """Export leaderboard in the specified format."""
+    lb = load_leaderboard(path)
+    content = export_leaderboard(lb, format=format)
+
+    if output:
+        Path(output).write_text(content)
+        console.print(f"Exported to: {output}")
+    else:
+        console.print(content)
+
+
+if __name__ == "__main__":
+    main()

bench_agent/evaluator.py

@@ -0,0 +1,129 @@
+"""Model evaluation orchestrator for BenchAgent benchmark."""
+
+from __future__ import annotations
+
+import json
+import time
+from typing import Any
+
+from bench_agent.models import ScoreReport, Task, TaskCategory, TaskResult
+from bench_agent.runner import TaskRunner
+from bench_agent.scorer import (
+    calculate_category_scores,
+    calculate_overall_score,
+    error_recovery_score,
+)
+from bench_agent.tasks import ALL_TASKS, TASKS_BY_CATEGORY
+
+PROMPT_TEMPLATE = """You are an AI assistant being evaluated on your tool-use capabilities.
+
+You have access to the following tools: {tools}
+
+Your task: {description}
+
+Working directory: {workdir}
+
+{initial_files}
+
+You must complete this task using the tools available. You have {max_turns} turns maximum.
+Be precise and thorough. After completing the task, verify your work.
+"""
+
+
+class ModelProvider:
+    OPENAI = "openai"
+    ANTHROPIC = "anthropic"
+    LOCAL = "local"
+    HUGGINGFACE = "huggingface"
+
+
+def _build_prompt(task: Task, workdir: str) -> str:
+    files_section = ""
+    if task.initial_state:
+        files_section = "Initial files:\n"
+        for fname, content in task.initial_state.items():
+            files_section += f"\n--- {fname} ---\n{content}\n"
+
+    return PROMPT_TEMPLATE.format(
+        tools=", ".join(task.tools_required),
+        description=task.description,
+        workdir=workdir,
+        initial_files=files_section,
+        max_turns=task.max_turns,
+    )
+
+
+def evaluate_model(
+    model_name: str,
+    provider: str = ModelProvider.OPENAI,
+    categories: list[TaskCategory] | None = None,
+    num_tasks: int | None = None,
+    api_key: str | None = None,
+    base_url: str | None = None,
+    runner: TaskRunner | None = None,
+) -> ScoreReport:
+    runner = runner or TaskRunner()
+
+    tasks: list[Task] = []
+    if categories:
+        for cat in categories:
+            tasks.extend(TASKS_BY_CATEGORY.get(cat, []))
+    else:
+        tasks = list(ALL_TASKS)
+
+    if num_tasks:
+        tasks = tasks[:num_tasks]
+
+    results: list[TaskResult] = []
+    for task in tasks:
+        try:
+            result = runner.run_task(task, model_name)
+            results.append(result)
+        except Exception as e:
+            results.append(
+                TaskResult(
+                    task_id=task.task_id,
+                    model=model_name,
+                    success=False,
+                    errors=[str(e)],
+                )
+            )
+
+    total_score = calculate_overall_score(results)
+    category_scores = calculate_category_scores(results)
+
+    recovery_scores = [error_recovery_score(r) for r in results]
+    avg_recovery = sum(recovery_scores) / len(recovery_scores) if recovery_scores else 0.0
+
+    return ScoreReport(
+        model=model_name,
+        total_score=total_score,
+        category_scores=category_scores,
+        error_recovery_rate=round(avg_recovery, 3),
+    )
+
+
+def evaluate_with_retry(
+    model_name: str,
+    provider: str = ModelProvider.OPENAI,
+    categories: list[TaskCategory] | None = None,
+    num_tasks: int | None = None,
+    max_retries: int = 3,
+    retry_delay: float = 5.0,
+    **kwargs: Any,
+) -> ScoreReport:
+    for attempt in range(max_retries):
+        try:
+            return evaluate_model(
+                model_name=model_name,
+                provider=provider,
+                categories=categories,
+                num_tasks=num_tasks,
+                **kwargs,
+            )
+        except Exception as e:
+            if attempt == max_retries - 1:
+                raise
+            time.sleep(retry_delay * (2**attempt))
+
+    raise RuntimeError("Unreachable")

bench_agent/leaderboard.py

@@ -0,0 +1,99 @@
+"""Leaderboard management for BenchAgent benchmark results."""
+
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+
+from bench_agent.models import Leaderboard, ScoreReport, TaskResult
+from bench_agent.scorer import calculate_overall_score, calculate_category_scores, error_recovery_score
+
+
+def load_leaderboard(path: str | Path) -> Leaderboard:
+    path = Path(path)
+    if not path.exists():
+        return Leaderboard(last_updated=datetime.now(timezone.utc).isoformat())
+
+    data = json.loads(path.read_text())
+    return Leaderboard(**data)
+
+
+def update_leaderboard(
+    leaderboard: Leaderboard, model_name: str, results: list[TaskResult]
+) -> Leaderboard:
+    total_score = calculate_overall_score(results)
+    category_scores = calculate_category_scores(results)
+
+    recovery_scores = [error_recovery_score(r) for r in results]
+    avg_recovery = sum(recovery_scores) / len(recovery_scores) if recovery_scores else 0.0
+
+    existing = None
+    for entry in leaderboard.entries:
+        if entry.model == model_name:
+            existing = entry
+            break
+
+    report = ScoreReport(
+        model=model_name,
+        total_score=total_score,
+        category_scores=category_scores,
+        error_recovery_rate=round(avg_recovery, 3),
+    )
+
+    if existing:
+        idx = leaderboard.entries.index(existing)
+        leaderboard.entries[idx] = report
+    else:
+        leaderboard.entries.append(report)
+
+    leaderboard = sort_leaderboard(leaderboard)
+    leaderboard.last_updated = datetime.now(timezone.utc).isoformat()
+    return leaderboard
+
+
+def sort_leaderboard(leaderboard: Leaderboard) -> Leaderboard:
+    sorted_entries = sorted(leaderboard.entries, key=lambda e: e.total_score, reverse=True)
+    for idx, entry in enumerate(sorted_entries):
+        entry.leaderboard_rank = idx + 1
+    leaderboard.entries = sorted_entries
+    return leaderboard
+
+
+def export_leaderboard(leaderboard: Leaderboard, format: str = "json") -> str:
+    if format == "json":
+        return leaderboard.model_dump_json(indent=2)
+    elif format == "markdown":
+        return export_markdown(leaderboard)
+    else:
+        raise ValueError(f"Unknown format: {format}")
+
+
+def export_markdown(leaderboard: Leaderboard) -> str:
+    lines = [
+        "# BenchAgent Leaderboard",
+        "",
+        f"Last updated: {leaderboard.last_updated}",
+        "",
+        "| Rank | Model | Total Score | Error Recovery | Bash | Edit | Read | Write | Multi-Tool | Error Recovery Cat |",
+        "|------|-------|-------------|-----------------|------|------|------|-------|------------|-------------------|",
+    ]
+
+    for entry in leaderboard.entries:
+        cat = entry.category_scores
+        lines.append(
+            f"| {entry.leaderboard_rank} | {entry.model} | {entry.total_score:.1f} | "
+            f"{entry.error_recovery_rate:.3f} | "
+            f"{cat.get('bash', 0):.1f} | {cat.get('edit', 0):.1f} | "
+            f"{cat.get('read', 0):.1f} | {cat.get('write', 0):.1f} | "
+            f"{cat.get('multi_tool', 0):.1f} | {cat.get('error_recovery', 0):.1f} |"
+        )
+
+    lines.append("")
+    return "\n".join(lines)
+
+
+def save_leaderboard(leaderboard: Leaderboard, path: str | Path) -> None:
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(export_leaderboard(leaderboard, format="json"))

bench_agent/models.py

@@ -0,0 +1,67 @@
+"""Pydantic models for BenchAgent benchmark definitions and results."""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class TaskCategory(str, Enum):
+    BASH = "bash"
+    EDIT = "edit"
+    READ = "read"
+    WRITE = "write"
+    MULTI_TOOL = "multi_tool"
+    ERROR_RECOVERY = "error_recovery"
+
+
+class Difficulty(str, Enum):
+    EASY = "easy"
+    MEDIUM = "medium"
+    HARD = "hard"
+
+
+class Task(BaseModel):
+    task_id: str
+    category: TaskCategory
+    difficulty: Difficulty
+    description: str
+    initial_state: dict[str, str] = Field(
+        default_factory=dict,
+        description="Mapping of filename -> file content to create before the task",
+    )
+    expected_outcome: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Expected result: keys may include 'files', 'exit_code', 'stdout', 'file_exists'",
+    )
+    tools_required: list[str] = Field(default_factory=list)
+    max_turns: int = 10
+    verification_script: str = ""
+
+
+class TaskResult(BaseModel):
+    task_id: str
+    model: str
+    success: bool = False
+    turns_used: int = 0
+    tokens_used: int = 0
+    errors: list[str] = Field(default_factory=list)
+    recovery_attempts: int = 0
+    duration_seconds: float = 0.0
+    actual_output: dict[str, Any] = Field(default_factory=dict)
+
+
+class ScoreReport(BaseModel):
+    model: str
+    total_score: float = 0.0
+    category_scores: dict[str, float] = Field(default_factory=dict)
+    tool_scores: dict[str, float] = Field(default_factory=dict)
+    error_recovery_rate: float = 0.0
+    leaderboard_rank: int = 0
+
+
+class Leaderboard(BaseModel):
+    entries: list[ScoreReport] = Field(default_factory=list)
+    last_updated: str = ""

bench_agent/runner.py

@@ -0,0 +1,159 @@
+"""Task execution runner with sandboxing, timeouts, and token tracking."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+import tempfile
+import time
+from pathlib import Path
+from typing import Any
+
+from bench_agent.models import Task, TaskCategory, TaskResult
+from bench_agent.tasks import TASKS_BY_CATEGORY, ALL_TASKS
+
+import json
+
+class TaskRunner:
+    def __init__(
+        self,
+        sandbox_root: str | Path | None = None,
+        per_turn_timeout: float = 30.0,
+        total_timeout: float = 300.0,
+    ) -> None:
+        self.sandbox_root = Path(sandbox_root) if sandbox_root else Path(tempfile.mkdtemp())
+        self.per_turn_timeout = per_turn_timeout
+        self.total_timeout = total_timeout
+
+    def _setup_sandbox(self, task: Task, sandbox_dir: Path) -> Path:
+        sandbox_dir.mkdir(parents=True, exist_ok=True)
+        for filename, content in task.initial_state.items():
+            filepath = sandbox_dir / filename
+            filepath.parent.mkdir(parents=True, exist_ok=True)
+            filepath.write_text(content)
+        return sandbox_dir
+
+    def _cleanup_sandbox(self, sandbox_dir: Path) -> None:
+        if sandbox_dir.exists():
+            shutil.rmtree(sandbox_dir, ignore_errors=True)
+
+    def _verify_task(self, task: Task, sandbox_dir: Path) -> bool:
+        if not task.verification_script:
+            return self._verify_expected_outcome(task, sandbox_dir)
+        try:
+            result = subprocess.run(
+                ["python3", "-c", task.verification_script],
+                cwd=str(sandbox_dir),
+                capture_output=True,
+                text=True,
+                timeout=15,
+            )
+            return result.returncode == 0
+        except (subprocess.TimeoutExpired, Exception):
+            return False
+
+    def _verify_expected_outcome(self, task: Task, sandbox_dir: Path) -> bool:
+        expected = task.expected_outcome
+        if not expected:
+            return True
+
+        if "files" in expected:
+            for filename, expected_content in expected["files"].items():
+                filepath = sandbox_dir / filename
+                if not filepath.exists():
+                    return False
+                actual = filepath.read_text()
+                if actual.strip() != expected_content.strip():
+                    return False
+
+        if "file_exists" in expected:
+            for filename in expected["file_exists"]:
+                if not (sandbox_dir / filename).exists():
+                    return False
+
+        if "exit_code" in expected:
+            pass
+
+        return True
+
+    def _count_tokens(self, text: str) -> int:
+        return max(1, len(text) // 4)
+
+    def run_task(
+        self, task: Task, model: str, max_turns: int | None = None
+    ) -> TaskResult:
+        max_turns = max_turns or task.max_turns
+        sandbox_dir = self.sandbox_root / f"task_{task.task_id}"
+        errors: list[str] = []
+        recovery_attempts = 0
+        start_time = time.time()
+
+        try:
+            self._setup_sandbox(task, sandbox_dir)
+
+            turns_used = 1
+            tokens_used = 0
+
+            prompt = f"You are a coding assistant. Complete this task:\n\n{task.description}\n\n"
+            prompt += f"Working directory: {sandbox_dir}\n"
+            prompt += f"Tools allowed: {', '.join(task.tools_required)}\n"
+            prompt += f"Max turns: {max_turns}\n"
+
+            if task.initial_state:
+                prompt += "\nInitial files:\n"
+                for fname, content in task.initial_state.items():
+                    prompt += f"\n--- {fname} ---\n{content}\n"
+
+            tokens_used += self._count_tokens(prompt)
+
+            for turn in range(max_turns):
+                turns_used = turn + 1
+                if time.time() - start_time > self.total_timeout:
+                    errors.append(f"Total timeout exceeded after {turns_used} turns")
+                    break
+
+            success = self._verify_task(task, sandbox_dir)
+
+        except Exception as e:
+            errors.append(str(e))
+            success = False
+            turns_used = 1
+        finally:
+            self._cleanup_sandbox(sandbox_dir)
+
+        duration = time.time() - start_time
+
+        return TaskResult(
+            task_id=task.task_id,
+            model=model,
+            success=success,
+            turns_used=turns_used,
+            tokens_used=tokens_used,
+            errors=errors,
+            recovery_attempts=recovery_attempts,
+            duration_seconds=round(duration, 2),
+        )
+
+    def run_benchmark(
+        self,
+        model: str,
+        categories: list[TaskCategory] | None = None,
+        num_tasks: int | None = None,
+    ) -> list[TaskResult]:
+        if categories:
+            tasks: list[Task] = []
+            for cat in categories:
+                tasks.extend(TASKS_BY_CATEGORY.get(cat, []))
+        else:
+            tasks = list(ALL_TASKS)
+
+        if num_tasks:
+            tasks = tasks[:num_tasks]
+
+        results: list[TaskResult] = []
+        for task in tasks:
+            result = self.run_task(task, model)
+            results.append(result)
+
+        return results