agentbridge-bench 0.1.0__py3-none-any.whl

agentbridge_bench/__init__.py

@@ -0,0 +1,5 @@
+"""AgentBench: reproducible evaluation of AI coding agents (package ``agentbridge-bench``)."""
+
+__version__ = "0.1.0"
+
+__all__ = ["__version__"]

agentbridge_bench/cli.py

@@ -0,0 +1,118 @@
+"""AgentBench command line: ``agentbridge-bench ...`` and ``agentbridge benchmark ...``."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich.markup import escape
+from rich.table import Table
+
+from agentbridge.cli.context import handle_errors
+from agentbridge.cli.output import console
+from agentbridge.errors import ConfigError, NotFoundError
+from agentbridge_bench.dataset import (
+    default_benchmarks_dir,
+    load_configurations,
+    load_suites,
+    load_tasks,
+)
+from agentbridge_bench.reports import generate_reports
+from agentbridge_bench.runner import run_benchmark
+
+bench_app = typer.Typer(help="AgentBench: reproducible agent evaluation.", no_args_is_help=True)
+
+DirOption = Annotated[
+    Path | None,
+    typer.Option("--benchmarks-dir", help="Directory with tasks/ and configurations.yaml"),
+]
+
+
+@bench_app.command("list")
+@handle_errors
+def bench_list(benchmarks_dir: DirOption = None) -> None:
+    """List benchmark tasks, suites and configurations."""
+    root = benchmarks_dir or default_benchmarks_dir()
+    tasks = load_tasks(root)
+    table = Table(title=f"AgentBench tasks ({root})")
+    for column in ("Task", "Category", "Difficulty", "Timeout (s)"):
+        table.add_column(column)
+    for task in tasks.values():
+        table.add_row(task.task_id, task.task_category.value, task.difficulty, str(task.timeout))
+    console.print(table)
+    for name, ids in load_suites(root, tasks).items():
+        console.print(f"suite [bold]{name}[/bold]: {len(ids)} task(s)")
+    for name, config in load_configurations(root).items():
+        console.print(
+            f"configuration [bold]{name}[/bold] ({config.kind}): {escape(config.description)}"
+        )
+
+
+@bench_app.command("run")
+@handle_errors
+def bench_run(
+    suite_name: Annotated[str, typer.Argument(help="Suite name, or a single task id.")],
+    config: Annotated[
+        list[str] | None,
+        typer.Option("--config", "-c", help="Configuration(s) to run (repeatable)."),
+    ] = None,
+    repetitions: Annotated[int, typer.Option("--repetitions", "-n", min=1, max=20)] = 1,
+    output: Annotated[Path | None, typer.Option("--output", help="Experiments directory.")] = None,
+    benchmarks_dir: DirOption = None,
+) -> None:
+    """Run a benchmark suite under one or more configurations."""
+    root = benchmarks_dir or default_benchmarks_dir()
+    tasks = load_tasks(root)
+    suites = load_suites(root, tasks)
+    if suite_name in suites:
+        selected = [tasks[i] for i in suites[suite_name]]
+    elif suite_name in tasks:
+        selected = [tasks[suite_name]]
+    else:
+        raise NotFoundError(
+            f"Unknown suite or task: {suite_name}", hint=f"Suites: {', '.join(suites)}"
+        )
+    configurations = load_configurations(root)
+    names = config or ["mock-oracle"]
+    unknown = [n for n in names if n not in configurations]
+    if unknown:
+        raise ConfigError(
+            f"Unknown configuration(s): {', '.join(unknown)}",
+            hint=f"Available: {', '.join(configurations)}",
+        )
+    exp_dir = run_benchmark(
+        root,
+        output or (root / "experiments"),
+        suite_name,
+        selected,
+        [configurations[n] for n in names],
+        repetitions,
+        progress=lambda msg: console.print(escape(msg)),
+    )
+    console.print(f"[green]Done.[/green] Report: {exp_dir / 'report.md'}")
+
+
+@bench_app.command("report")
+@handle_errors
+def bench_report(
+    experiment_dir: Annotated[Path, typer.Argument(help="Experiment directory.")],
+) -> None:
+    """Regenerate CSV/Markdown/SVG reports from saved experiment records."""
+    if not (experiment_dir / "records").is_dir():
+        raise NotFoundError(f"No records/ directory in {experiment_dir}")
+    summary = generate_reports(experiment_dir)
+    for name, metrics in summary.items():
+        console.print(
+            f"{name}: success={metrics['task_success_rate']} executed={metrics['runs_executed']}"
+        )
+    console.print(f"Report: {experiment_dir / 'report.md'}")
+
+
+def main() -> None:
+    """Entry point of the ``agentbridge-bench`` command."""
+    bench_app(prog_name="agentbridge-bench")
+
+
+if __name__ == "__main__":
+    main()

agentbridge_bench/data/configurations.yaml

@@ -0,0 +1,65 @@
+# AgentBench configurations.
+#
+# A configuration is only executed when every provider it uses is available and
+# can run without a human. Others are recorded as "not_executable" (never as
+# failures and never as fabricated results).
+
+configurations:
+  mock-oracle:
+    description: >-
+      Harness control. The mock provider writes the task's reference solution, so
+      every task should pass. Validates fixtures and hidden tests; says nothing
+      about AI agents.
+    kind: control
+    planner_mode: single
+    oracle: true
+    providers:
+      mock: {type: mock}
+    routing: {planner: mock, default_worker: mock}
+
+  mock-noop:
+    description: >-
+      Negative control. The mock provider writes an unrelated file, so tasks
+      should fail. Confirms the hidden tests cannot be passed without a change.
+    kind: control
+    planner_mode: single
+    providers:
+      mock: {type: mock}
+    routing: {planner: mock, default_worker: mock}
+
+  claude-single:
+    description: "Configuration A: single Claude Code agent (Task -> Claude Code -> Result)."
+    kind: single_agent
+    planner_mode: single
+    providers:
+      claude: {type: claude_code, command: claude, permission_mode: acceptEdits}
+    routing: {planner: claude, default_worker: claude}
+
+  claude-multi:
+    description: >-
+      Multi-agent workflow using Claude Code in three roles: analyst -> developer ->
+      reviewer (template plan), with an additional agent review of the developer diff.
+    kind: multi_agent
+    planner_mode: standard
+    providers:
+      claude: {type: claude_code, command: claude, permission_mode: acceptEdits}
+    routing: {planner: claude, default_worker: claude, reviewer: claude}
+
+  antigravity-remote:
+    description: "Configuration B: single Antigravity remote agent (manual handoff)."
+    kind: remote
+    planner_mode: single
+    providers:
+      antigravity: {type: antigravity_remote, command: agy}
+    routing: {planner: antigravity, default_worker: antigravity}
+
+  claude-antigravity:
+    description: >-
+      Configuration C: Claude planner -> Antigravity remote worker -> independent
+      reviewer. Requires a manual remote handoff.
+    kind: remote
+    planner_mode: provider
+    providers:
+      claude: {type: claude_code, command: claude, permission_mode: acceptEdits}
+      antigravity: {type: antigravity_remote, command: agy}
+    routing: {planner: claude, default_worker: antigravity, reviewer: claude}

agentbridge_bench/data/suites.yaml

@@ -0,0 +1,19 @@
+# Named task suites. "all" (every task) is always available.
+suites:
+  core:
+    - analysis-001
+    - bugfix-001
+    - bugfix-002
+    - bugfix-003
+    - deps-001
+    - docs-001
+    - feature-001
+    - feature-002
+    - feature-003
+    - refactor-001
+    - review-001
+    - tests-001
+  smoke:
+    - bugfix-001
+    - feature-001
+    - tests-001

agentbridge_bench/data/tasks/analysis-001/hidden_tests/test_hidden_analysis.py

@@ -0,0 +1,33 @@
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def text():
+    path = ROOT / "ANALYSIS.md"
+    assert path.exists(), "ANALYSIS.md was not written"
+    return path.read_text(encoding="utf-8")
+
+
+def test_modules_listed():
+    content = text()
+    for module in ("cart", "pricing", "tax", "cli"):
+        assert module in content
+
+
+def test_functions_listed():
+    content = text()
+    for name in ("add_item", "remove_item", "item_count", "apply_discount", "round_price",
+                 "tax_for"):
+        assert name in content
+
+
+def test_entry_point():
+    assert "main" in text()
+
+
+def test_untested_modules():
+    lines = [line.lower() for line in text().splitlines() if "untested" in line.lower()]
+    for module in ("pricing", "tax", "cli"):
+        assert any(module in line for line in lines), module
+    assert not any("cart" in line for line in lines)

agentbridge_bench/data/tasks/analysis-001/repo/pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+addopts = -p no:cacheprovider
+testpaths = tests

agentbridge_bench/data/tasks/analysis-001/repo/shop/__init__.py

@@ -0,0 +1 @@
+"""A tiny shop."""

agentbridge_bench/data/tasks/analysis-001/repo/shop/cart.py

@@ -0,0 +1,15 @@
+"""Shopping cart."""
+
+
+def add_item(cart, sku, quantity=1):
+    cart[sku] = cart.get(sku, 0) + quantity
+    return cart
+
+
+def remove_item(cart, sku):
+    cart.pop(sku, None)
+    return cart
+
+
+def item_count(cart):
+    return sum(cart.values())

agentbridge_bench/data/tasks/analysis-001/repo/shop/cli.py

@@ -0,0 +1,18 @@
+"""Command line interface."""
+
+import sys
+
+from shop.cart import add_item, item_count
+
+
+def main(argv=None):
+    args = sys.argv[1:] if argv is None else argv
+    cart = {}
+    for sku in args:
+        add_item(cart, sku)
+    print(item_count(cart))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

agentbridge_bench/data/tasks/analysis-001/repo/shop/pricing.py

@@ -0,0 +1,9 @@
+"""Prices and discounts."""
+
+
+def apply_discount(price, percent):
+    return price * (100 - percent) / 100
+
+
+def round_price(price):
+    return round(price, 2)

agentbridge_bench/data/tasks/analysis-001/repo/shop/tax.py

@@ -0,0 +1,7 @@
+"""Sales tax."""
+
+TAX_RATES = {"standard": 0.2, "reduced": 0.05}
+
+
+def tax_for(amount, category="standard"):
+    return amount * TAX_RATES[category]

agentbridge_bench/data/tasks/analysis-001/repo/tests/test_cart.py

@@ -0,0 +1,5 @@
+from shop.cart import add_item, item_count
+
+
+def test_add():
+    assert item_count(add_item({}, "a", 2)) == 2

agentbridge_bench/data/tasks/analysis-001/solution/ANALYSIS.md

@@ -0,0 +1,20 @@
+# Repository analysis
+
+## Modules in `shop`
+
+- `shop.cart`: add_item, remove_item, item_count
+- `shop.pricing`: apply_discount, round_price
+- `shop.tax`: tax_for (plus the TAX_RATES table)
+- `shop.cli`: main
+
+## Entry point
+
+The command-line entry point is `shop.cli.main` (run `python -m shop.cli`).
+
+## Test coverage
+
+Only `shop.cart` has tests (tests/test_cart.py).
+
+- pricing: untested
+- tax: untested
+- cli: untested

agentbridge_bench/data/tasks/analysis-001/task.yaml

@@ -0,0 +1,30 @@
+# base_commit is null for fixture tasks: the base commit is synthesised
+# deterministically from repo/ (see docs/benchmark-methodology.md).
+task_id: analysis-001
+repository: fixture
+base_commit: null
+task_category: repository_analysis
+difficulty: easy
+timeout: 900
+issue_description: |
+  Write ANALYSIS.md describing this repository: list every module in the shop package together with its public functions, identify the command-line entry point, and state which modules have no tests (one line per untested module, containing the module name and the word "untested"). Do not modify source files.
+acceptance_criteria:
+- ANALYSIS.md lists all shop modules and their public functions
+- ANALYSIS.md names the CLI entry point
+- ANALYSIS.md lists the untested modules
+expected_files:
+- ANALYSIS.md
+setup_commands: []
+test_commands:
+- - '{python}'
+  - -m
+  - pytest
+  - -q
+  - agentbench_hidden
+regression_test_command:
+- '{python}'
+- -m
+- pytest
+- -q
+- tests
+license: MIT (authored for AgentBench)

agentbridge_bench/data/tasks/bugfix-001/hidden_tests/test_hidden_slugify.py

@@ -0,0 +1,17 @@
+from textutils import slugify
+
+
+def test_collapses_separators():
+    assert slugify("Hello,  World!") == "hello-world"
+
+
+def test_strips_edges():
+    assert slugify("  --Python 3.12--  ") == "python-3-12"
+
+
+def test_empty():
+    assert slugify("") == ""
+
+
+def test_plain():
+    assert slugify("abc") == "abc"

agentbridge_bench/data/tasks/bugfix-001/repo/pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+addopts = -p no:cacheprovider
+testpaths = tests

agentbridge_bench/data/tasks/bugfix-001/repo/tests/test_textutils.py

@@ -0,0 +1,9 @@
+from textutils import slugify
+
+
+def test_simple_word():
+    assert slugify("hello") == "hello"
+
+
+def test_lowercases():
+    assert slugify("Hello") == "hello"

agentbridge_bench/data/tasks/bugfix-001/repo/textutils.py

@@ -0,0 +1,9 @@
+"""Small text helpers."""
+
+import re
+
+
+def slugify(text: str) -> str:
+    """Convert text to a URL slug: lowercase words joined by single dashes."""
+    text = text.lower()
+    return re.sub(r"[^a-z0-9]", "-", text)

agentbridge_bench/data/tasks/bugfix-001/solution/textutils.py

@@ -0,0 +1,8 @@
+"""Small text helpers."""
+
+import re
+
+
+def slugify(text: str) -> str:
+    """Convert text to a URL slug: lowercase words joined by single dashes."""
+    return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")

agentbridge_bench/data/tasks/bugfix-001/task.yaml

@@ -0,0 +1,30 @@
+# base_commit is null for fixture tasks: the base commit is synthesised
+# deterministically from repo/ (see docs/benchmark-methodology.md).
+task_id: bugfix-001
+repository: fixture
+base_commit: null
+task_category: bug_fixing
+difficulty: easy
+timeout: 900
+issue_description: |
+  textutils.slugify() produces malformed slugs. slugify("Hello, World!") returns "hello---world-" but should return "hello-world". Runs of non-alphanumeric characters must collapse into a single dash, and the slug must not start or end with a dash.
+acceptance_criteria:
+- slugify collapses runs of separators into one dash
+- slugify strips leading and trailing dashes
+- Existing tests keep passing
+expected_files:
+- textutils.py
+setup_commands: []
+test_commands:
+- - '{python}'
+  - -m
+  - pytest
+  - -q
+  - agentbench_hidden
+regression_test_command:
+- '{python}'
+- -m
+- pytest
+- -q
+- tests
+license: MIT (authored for AgentBench)

agentbridge_bench/data/tasks/bugfix-002/hidden_tests/test_hidden_median.py

@@ -0,0 +1,24 @@
+import pytest
+
+from stats import mean, median
+
+
+def test_even():
+    assert median([1, 2, 3, 4]) == 2.5
+
+
+def test_even_unsorted():
+    assert median([4, 1, 3, 2]) == 2.5
+
+
+def test_single():
+    assert median([5]) == 5
+
+
+def test_empty_raises_value_error():
+    with pytest.raises(ValueError):
+        median([])
+
+
+def test_mean_unchanged():
+    assert mean([2, 4]) == 3

agentbridge_bench/data/tasks/bugfix-002/repo/pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+addopts = -p no:cacheprovider
+testpaths = tests

agentbridge_bench/data/tasks/bugfix-002/repo/stats.py

@@ -0,0 +1,15 @@
+"""Basic statistics."""
+
+
+def mean(values):
+    """Arithmetic mean of a non-empty sequence."""
+    values = list(values)
+    if not values:
+        raise ValueError("mean() of empty sequence")
+    return sum(values) / len(values)
+
+
+def median(values):
+    """Median of a non-empty sequence of numbers."""
+    ordered = sorted(values)
+    return ordered[len(ordered) // 2]

agentbridge_bench/data/tasks/bugfix-002/repo/tests/test_stats.py

@@ -0,0 +1,9 @@
+from stats import mean, median
+
+
+def test_mean():
+    assert mean([1, 2, 3]) == 2
+
+
+def test_median_odd():
+    assert median([3, 1, 2]) == 2

agentbridge_bench/data/tasks/bugfix-002/solution/stats.py

@@ -0,0 +1,20 @@
+"""Basic statistics."""
+
+
+def mean(values):
+    """Arithmetic mean of a non-empty sequence."""
+    values = list(values)
+    if not values:
+        raise ValueError("mean() of empty sequence")
+    return sum(values) / len(values)
+
+
+def median(values):
+    """Median of a non-empty sequence of numbers."""
+    ordered = sorted(values)
+    if not ordered:
+        raise ValueError("median() of empty sequence")
+    middle = len(ordered) // 2
+    if len(ordered) % 2:
+        return ordered[middle]
+    return (ordered[middle - 1] + ordered[middle]) / 2

agentbridge_bench/data/tasks/bugfix-002/task.yaml

@@ -0,0 +1,30 @@
+# base_commit is null for fixture tasks: the base commit is synthesised
+# deterministically from repo/ (see docs/benchmark-methodology.md).
+task_id: bugfix-002
+repository: fixture
+base_commit: null
+task_category: bug_fixing
+difficulty: easy
+timeout: 900
+issue_description: |
+  stats.median() returns the wrong value for sequences with an even number of elements: median([1, 2, 3, 4]) returns 3 instead of 2.5. It should return the mean of the two middle values. median([]) currently raises IndexError; it must raise ValueError instead.
+acceptance_criteria:
+- median returns the mean of the two middle values for even-length input
+- median([]) raises ValueError
+- Existing tests keep passing
+expected_files:
+- stats.py
+setup_commands: []
+test_commands:
+- - '{python}'
+  - -m
+  - pytest
+  - -q
+  - agentbench_hidden
+regression_test_command:
+- '{python}'
+- -m
+- pytest
+- -q
+- tests
+license: MIT (authored for AgentBench)

agentbridge_bench/data/tasks/bugfix-003/hidden_tests/test_hidden_pagination.py

@@ -0,0 +1,27 @@
+import pytest
+
+from pagination import page_count, paginate
+
+ITEMS = list(range(25))
+
+
+def test_first_page():
+    assert paginate(ITEMS, 1, 10) == list(range(10))
+
+
+def test_last_partial_page():
+    assert paginate(ITEMS, 3, 10) == [20, 21, 22, 23, 24]
+
+
+def test_page_count_rounds_up():
+    assert page_count(25, 10) == 3
+
+
+def test_page_count_exact_and_zero():
+    assert page_count(20, 10) == 2
+    assert page_count(0, 10) == 0
+
+
+def test_invalid_page():
+    with pytest.raises(ValueError):
+        paginate(ITEMS, 0)

agentbridge_bench/data/tasks/bugfix-003/repo/pagination.py

@@ -0,0 +1,12 @@
+"""List pagination helpers."""
+
+
+def paginate(items, page, per_page=10):
+    """Return the items on a 1-indexed page."""
+    start = page * per_page
+    return items[start : start + per_page]
+
+
+def page_count(total, per_page=10):
+    """Number of pages needed for ``total`` items."""
+    return total // per_page

agentbridge_bench/data/tasks/bugfix-003/repo/pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+addopts = -p no:cacheprovider
+testpaths = tests

agentbridge_bench/data/tasks/bugfix-003/repo/tests/test_pagination.py

@@ -0,0 +1,5 @@
+from pagination import page_count
+
+
+def test_exact_pages():
+    assert page_count(20, 10) == 2

agentbridge_bench/data/tasks/bugfix-003/solution/pagination.py

@@ -0,0 +1,14 @@
+"""List pagination helpers."""
+
+
+def paginate(items, page, per_page=10):
+    """Return the items on a 1-indexed page."""
+    if page < 1:
+        raise ValueError("page numbers start at 1")
+    start = (page - 1) * per_page
+    return items[start : start + per_page]
+
+
+def page_count(total, per_page=10):
+    """Number of pages needed for ``total`` items."""
+    return -(-total // per_page)

agentbridge_bench/data/tasks/bugfix-003/task.yaml

@@ -0,0 +1,31 @@
+# base_commit is null for fixture tasks: the base commit is synthesised
+# deterministically from repo/ (see docs/benchmark-methodology.md).
+task_id: bugfix-003
+repository: fixture
+base_commit: null
+task_category: bug_fixing
+difficulty: medium
+timeout: 900
+issue_description: |
+  pagination.paginate() is documented as 1-indexed but paginate(items, 1) returns the second page. page_count(25, 10) returns 2 but there are 3 pages. Fix both, and make paginate() raise ValueError for page numbers below 1.
+acceptance_criteria:
+- paginate(items, 1) returns the first page
+- page_count rounds up partial pages
+- paginate raises ValueError for page < 1
+- Existing tests keep passing
+expected_files:
+- pagination.py
+setup_commands: []
+test_commands:
+- - '{python}'
+  - -m
+  - pytest
+  - -q
+  - agentbench_hidden
+regression_test_command:
+- '{python}'
+- -m
+- pytest
+- -q
+- tests
+license: MIT (authored for AgentBench)