benchkit-cli 0.1.0__py3-none-any.whl

benchkit/benchmarks/aider_polyglot.py

@@ -0,0 +1,405 @@
+"""Aider Polyglot exercises executed by the stock Pi coding agent."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import shutil
+import subprocess
+import threading
+from pathlib import Path
+
+from benchkit.benchmarks.base import Task
+from benchkit.benchmarks.downloads import Dataset
+from benchkit.evaluation import EvaluationResult
+from benchkit.sandbox import (
+    AIDER_POLYGLOT_COMMIT,
+    DockerTaskEnvironment,
+    aider_pi_image,
+)
+
+_REPOSITORY = "https://github.com/Aider-AI/polyglot-benchmark.git"
+_LANGUAGES = ("cpp", "go", "java", "javascript", "python", "rust")
+_LANGUAGE_ALIASES = {
+    "cpp": "cpp",
+    "c++": "cpp",
+    "go": "go",
+    "java": "java",
+    "js": "javascript",
+    "javascript": "javascript",
+    "py": "python",
+    "python": "python",
+    "rs": "rust",
+    "rust": "rust",
+}
+_EXPECTED_COUNTS = {
+    "cpp": 26,
+    "go": 39,
+    "java": 47,
+    "javascript": 49,
+    "python": 34,
+    "rust": 30,
+}
+_TEST_COMMANDS = {
+    "cpp": [
+        "bash",
+        "-lc",
+        "timeout --signal=KILL 120s bash -lc "
+        "'cmake -S . -B build -DEXERCISM_RUN_ALL_TESTS=ON "
+        "&& cmake --build build --parallel 2'",
+    ],
+    "go": ["go", "test", "./..."],
+    "java": [
+        "gradle",
+        "-Dfile.encoding=UTF-8",
+        "test",
+        "--offline",
+        "--no-daemon",
+    ],
+    "javascript": ["npm", "test", "--", "--runInBand"],
+    "python": ["python3", "-m", "unittest", "discover", "-v", "-p", "*_test.py"],
+    "rust": ["cargo", "test", "--offline"],
+}
+_ENABLE_TEST_COMMANDS = {
+    "java": [
+        "bash",
+        "-lc",
+        "find src/test -name '*.java' -exec "
+        "sed -i '/^[[:space:]]*@Disabled/d;"
+        "/import org.junit.jupiter.api.Disabled;/d' {} +",
+    ],
+    "javascript": [
+        "bash",
+        "-lc",
+        "find . -maxdepth 1 -name '*.spec.js' -exec "
+        "sed -i -E 's/\\b(xit|xtest)\\s*\\(/test(/g; "
+        "s/\\b(xdescribe)\\s*\\(/describe(/g; "
+        "s/\\b(test|it|describe)\\.skip\\s*\\(/\\1(/g' {} +",
+    ],
+    "rust": [
+        "bash",
+        "-lc",
+        "find tests -name '*.rs' -exec "
+        "sed -i '/^[[:space:]]*#\\[ignore[^]]*\\][[:space:]]*$/d' {} +",
+    ],
+}
+_SAFE_NAME = re.compile(r"^[a-z0-9][a-z0-9-]*$")
+_CACHE_LOCK = threading.Lock()
+_ANSWER_KEY_GUARD_LOG = "/tmp/benchkit-answer-key-guard.jsonl"
+
+
+def _cache_root() -> Path:
+    configured = os.environ.get("BENCHKIT_AIDER_DATASET")
+    if configured:
+        return Path(configured).expanduser()
+    return Path.home() / ".cache" / "benchkit" / "aider-polyglot"
+
+
+def _run_git(args: list[str], *, timeout: float = 300) -> None:
+    git = shutil.which("git")
+    if git is None:
+        raise RuntimeError("Aider Polyglot requires git to download its task corpus")
+    completed = subprocess.run(
+        [git, *args],
+        text=True,
+        capture_output=True,
+        timeout=timeout,
+    )
+    if completed.returncode:
+        detail = (completed.stderr or completed.stdout).strip()[-2000:]
+        raise RuntimeError(f"could not prepare Aider Polyglot dataset: {detail}")
+
+
+def _dataset_root() -> Path:
+    """Return a pinned upstream checkout, downloading it once when necessary."""
+    root = _cache_root()
+    marker = root / ".git"
+    with _CACHE_LOCK:
+        if not marker.exists():
+            root.parent.mkdir(parents=True, exist_ok=True)
+            _run_git(["clone", _REPOSITORY, str(root)])
+        completed = subprocess.run(
+            [shutil.which("git") or "git", "-C", str(root), "rev-parse", "HEAD"],
+            text=True,
+            capture_output=True,
+            timeout=30,
+        )
+        if completed.returncode or completed.stdout.strip() != AIDER_POLYGLOT_COMMIT:
+            _run_git(["-C", str(root), "fetch", "origin", AIDER_POLYGLOT_COMMIT])
+            _run_git(["-C", str(root), "checkout", "--detach", AIDER_POLYGLOT_COMMIT])
+    return root
+
+
+class AiderPolyglot:
+    """Run the complete Aider exercise corpus through Pi's native tools."""
+
+    name = "aider-polyglot"
+    task_count = sum(_EXPECTED_COUNTS.values())
+    data_source = Dataset(
+        summary="pinned Aider polyglot-benchmark checkout (needs git)",
+        # A full clone; git's packing makes the exact figure drift.
+        size=25_000_000,
+        approximate=True,
+        location=_cache_root,
+        is_installed=lambda: (_cache_root() / ".git").exists(),
+        install=lambda _progress: _dataset_root(),
+    )
+    variant_count = len(_LANGUAGES)
+    workspace_task = True
+    evaluation_activity = "running exercise tests in the Pi sandbox"
+
+    def variants(self, _client: object, _model: str) -> tuple[str, ...]:
+        return _LANGUAGES
+
+    def resolve_variant(self, selector: str) -> str | None:
+        """Resolve a CLI language selector to its canonical variant name."""
+        return _LANGUAGE_ALIASES.get(selector.strip().lower())
+
+    def variant_task_count(self, variant: str) -> int:
+        return _EXPECTED_COUNTS[variant]
+
+    def load_tasks(self) -> list[Task]:
+        root = _dataset_root()
+        tasks: list[Task] = []
+        for language in _LANGUAGES:
+            exercises = root / language / "exercises" / "practice"
+            names = sorted(path.name for path in exercises.iterdir() if path.is_dir())
+            expected = _EXPECTED_COUNTS[language]
+            if len(names) != expected:
+                raise RuntimeError(
+                    f"Aider Polyglot {language} checkout has {len(names)} tasks; "
+                    f"expected {expected} at {AIDER_POLYGLOT_COMMIT[:12]}"
+                )
+            tasks.extend(
+                Task(
+                    id=f"{language}/{name}",
+                    prompt="",
+                    metadata={"language": language, "exercise": name},
+                )
+                for name in names
+            )
+        return tasks
+
+    def tasks_for_variant(self, tasks: list[Task], variant: str) -> list[Task]:
+        return [task for task in tasks if task.metadata["language"] == variant]
+
+    def build_prompt(self, task: Task) -> str:
+        language = task.metadata["language"]
+        exercise = task.metadata["exercise"]
+        if language == "go" and exercise == "counter":
+            return (
+                "Solve the Aider Polyglot go exercise `counter` in the current "
+                "workspace. This is a test-authoring exercise: read "
+                "`.docs/instructions.md`, inspect the supplied implementations, "
+                "and write a comprehensive `counter_test.go` that accepts the "
+                "correct implementation and rejects every incorrect one. Use the "
+                "terminal to run checks yourself, bounding every command with "
+                "`timeout --signal=KILL 120s`. Do not modify the supplied "
+                "implementations or exercise instructions."
+            )
+        return (
+            f"Solve the Aider Polyglot {language} exercise `{exercise}` in the "
+            "current workspace. Read `.docs/instructions.md` and any appendices, "
+            "inspect the starter code and tests, then edit the implementation so "
+            "all tests pass. Use the terminal to run the tests yourself, but bound "
+            "every test command with `timeout --signal=KILL 120s` so a broken "
+            "implementation cannot deadlock the agent. Do not change the tests or "
+            "exercise instructions."
+        )
+
+    def evaluate(self, _task: Task, _response: str) -> bool:
+        raise RuntimeError("Aider Polyglot must be evaluated inside its workspace")
+
+    def pi_image(self):
+        return aider_pi_image()
+
+    def prepare_workspace(
+        self,
+        task: Task,
+        environment: DockerTaskEnvironment,
+    ) -> None:
+        language, exercise = self._coordinates(task)
+        workspace = f"/workspace/{exercise}"
+        environment.workdir = workspace
+        source = f"/opt/aider-polyglot/{language}/exercises/practice/{exercise}/."
+        environment.exec(["mkdir", "-p", workspace])
+        environment.exec(["cp", "-a", source, f"{workspace}/"])
+        environment.exec(
+            [
+                "find",
+                workspace,
+                "-depth",
+                "(",
+                "-name",
+                ".meta",
+                "-o",
+                "-name",
+                ".approaches",
+                "-o",
+                "-iname",
+                "*example*",
+                "-o",
+                "-iname",
+                "*reference*",
+                "-o",
+                "-iname",
+                "*proof*",
+                ")",
+                "-exec",
+                "rm",
+                "-rf",
+                "{}",
+                "+",
+            ]
+        )
+        environment.exec(["rm", "-rf", f"{workspace}/.git"])
+        environment.exec(["rm", "-f", _ANSWER_KEY_GUARD_LOG])
+        environment.exec(["git", "init", "-q", workspace])
+        environment.exec(["git", "-C", workspace, "config", "user.name", "BenchKit"])
+        environment.exec(
+            [
+                "git",
+                "-C",
+                workspace,
+                "config",
+                "user.email",
+                "benchkit@localhost",
+            ]
+        )
+        environment.exec(["git", "-C", workspace, "add", "."])
+        environment.exec(["git", "-C", workspace, "commit", "-q", "-m", "baseline"])
+
+    def verify_workspace(
+        self,
+        task: Task,
+        environment: DockerTaskEnvironment,
+        _tool_trace: list[dict] | None = None,
+    ) -> EvaluationResult:
+        language, exercise = self._coordinates(task)
+        workspace = f"/workspace/{exercise}"
+        command = _TEST_COMMANDS[language]
+        try:
+            patch = environment.exec(
+                ["git", "-C", workspace, "diff", "--no-ext-diff"],
+                timeout=30,
+                check=False,
+            ).stdout
+            changed = environment.exec(
+                ["git", "-C", workspace, "diff", "--name-only"],
+                timeout=30,
+                check=False,
+            ).stdout.splitlines()
+            changed_tests = [
+                path for path in changed if self._is_test_file(language, path)
+            ]
+            if language == "go" and exercise == "counter":
+                changed_tests = [
+                    path for path in changed_tests if path != "counter_test.go"
+                ]
+            if changed_tests:
+                environment.exec(
+                    [
+                        "git",
+                        "-C",
+                        workspace,
+                        "checkout",
+                        "HEAD",
+                        "--",
+                        *changed_tests,
+                    ],
+                    timeout=30,
+                )
+            enable_command = _ENABLE_TEST_COMMANDS.get(language)
+            if enable_command is not None:
+                environment.exec(
+                    enable_command,
+                    workdir=workspace,
+                    timeout=30,
+                )
+            if language == "go" and exercise == "counter":
+                command = [
+                    "bash",
+                    "-lc",
+                    "for implementation in 1 2 3; do "
+                    "if COUNTER_IMPL=$implementation go test ./...; then exit 1; fi; "
+                    "done; COUNTER_IMPL=4 go test ./...",
+                ]
+            completed = environment.exec(
+                command,
+                workdir=workspace,
+                timeout=180,
+                check=False,
+            )
+            guard_result = environment.exec(
+                ["cat", _ANSWER_KEY_GUARD_LOG], timeout=10, check=False
+            )
+            guard_hits = []
+            for line in guard_result.stdout.splitlines():
+                try:
+                    hit = json.loads(line)
+                except (ValueError, TypeError):
+                    continue
+                if isinstance(hit, dict):
+                    guard_hits.append(hit)
+        except Exception as exc:
+            return EvaluationResult(
+                0.0,
+                error=f"workspace verification failed: {type(exc).__name__}: {exc}",
+            )
+        output = (completed.stdout + completed.stderr).strip()
+        passed = completed.returncode == 0
+        contaminated = bool(guard_hits)
+        feedback = ""
+        if not passed:
+            feedback = (
+                "The exercise tests still fail. Review this test output, fix the "
+                f"workspace, and run the tests again:\n\n{output[-6000:]}"
+            )
+        return EvaluationResult(
+            score=1.0 if passed and not contaminated else 0.0,
+            feedback=feedback,
+            details={
+                "language": language,
+                "contamination_verdict": ("CONTAMINATED" if contaminated else "CLEAR"),
+                "answer_key_guard_hits": guard_hits,
+                "test_command": command,
+                "test_exit_code": completed.returncode,
+                "test_output": output[-8000:],
+                "test_output_truncated": len(output) > 8000,
+                "test_files_restored": changed_tests,
+                "patch": patch[-16000:],
+                "patch_truncated": len(patch) > 16000,
+            },
+        )
+
+    @staticmethod
+    def _coordinates(task: Task) -> tuple[str, str]:
+        language = str(task.metadata.get("language") or "")
+        exercise = str(task.metadata.get("exercise") or "")
+        if language not in _LANGUAGES or not _SAFE_NAME.fullmatch(exercise):
+            raise ValueError(f"invalid Aider Polyglot task coordinates: {task.id}")
+        return language, exercise
+
+    @staticmethod
+    def _is_test_file(language: str, path: str) -> bool:
+        name = Path(path).name
+        if language == "cpp":
+            return name.endswith("_test.cpp") or path.startswith("test/")
+        if language == "go":
+            return name.endswith("_test.go")
+        if language == "java":
+            return path.startswith("src/test/")
+        if language == "javascript":
+            return name.endswith(".spec.js")
+        if language == "python":
+            return name.endswith("_test.py")
+        return language == "rust" and path.startswith("tests/")
+
+    def result_metadata(self, variant: str | None) -> dict:
+        return {
+            "dataset_commit": AIDER_POLYGLOT_COMMIT,
+            "language": variant,
+            "protocol": "pi-agent",
+            "official_aider_comparable": False,
+        }

benchkit/benchmarks/arc.py

@@ -0,0 +1,52 @@
+"""ARC-Challenge benchmark - science multiple choice questions."""
+
+import json
+from pathlib import Path
+
+from benchkit.benchmarks.base import Task
+from benchkit.benchmarks.mcq import extract_choice
+from benchkit.benchmarks.utils import strip_think_tags
+
+DATASET = Path(__file__).parent.parent / "datasets" / "arc_challenge.jsonl"
+
+SYSTEM = (
+    "Answer the following multiple choice question. "
+    "Reply with ONLY the letter (A, B, C, or D)."
+)
+
+
+def _format_choices(choices: list[str]) -> str:
+    return "\n".join(
+        f"{letter}) {choice}" for letter, choice in zip("ABCD", choices, strict=False)
+    )
+
+
+class ARC:
+    name = "arc"
+    decision_instructions = "Which option correctly answers the question?"
+
+    def load_tasks(self) -> list[Task]:
+        tasks = []
+        with open(DATASET) as f:
+            for line in f:
+                d = json.loads(line)
+                tasks.append(
+                    Task(
+                        id=d["task_id"],
+                        prompt=d["question"],
+                        metadata={
+                            "choices": d["choices"],
+                            "answer": d["answer"],
+                        },
+                    )
+                )
+        return tasks
+
+    def build_prompt(self, task: Task) -> str:
+        choices = _format_choices(task.metadata["choices"])
+        return f"{SYSTEM}\n\n{task.prompt}\n\n{choices}"
+
+    def evaluate(self, task: Task, response: str) -> bool:
+        text = strip_think_tags(response)
+        choice = extract_choice(text, task.metadata["choices"])
+        return choice == task.metadata["answer"]

benchkit/benchmarks/banking77.py

@@ -0,0 +1,149 @@
+"""BANKING77 intent routing - classify a customer message into one of 77 intents.
+
+Routing is what decision models are deployed for, so the suite is shaped like
+a router: every task offers all 77 intents (no option filtering) and the answer
+is the intent label itself rather than an option letter. Chat models get the
+same list and are scored on an exact match of the label they reply with.
+"""
+
+import json
+from collections import Counter
+from functools import cache
+from pathlib import Path
+
+from benchkit.benchmarks.base import Task
+from benchkit.benchmarks.utils import strip_think_tags
+from benchkit.evaluation import EvaluationResult
+
+DATASETS = Path(__file__).parent.parent / "datasets"
+DATASET = DATASETS / "banking77.jsonl"
+INTENTS = DATASETS / "banking77_intents.json"
+
+SYSTEM = (
+    "Route the customer's message to exactly one of the 77 banking intents "
+    "below. Reply with ONLY the intent name, exactly as written."
+)
+
+# Quotes, emphasis and heading marks a model may put around an exact label.
+_WRAPPERS = " \t`'\"*#.,;:"
+
+
+@cache
+def intents() -> tuple[str, ...]:
+    """The 77 intent labels in upstream order."""
+    return tuple(json.loads(INTENTS.read_text(encoding="utf-8")))
+
+
+@cache
+def _gold() -> dict[str, str]:
+    """Expected intent by task id."""
+    with open(DATASET, encoding="utf-8") as f:
+        return {row["task_id"]: row["answer"] for row in map(json.loads, f)}
+
+
+def extract_intent(response: str) -> str | None:
+    """Return the intent a response names, or None.
+
+    The label must be exact up to case and surrounding quotes, backticks,
+    emphasis or heading marks, optionally after an ``intent:`` prefix. The whole
+    answer is tried first, then its last line, since a model may explain before
+    committing.
+    """
+    labels = {label.casefold(): label for label in intents()}
+    text = strip_think_tags(response).strip()
+    lines = [line for line in text.splitlines() if line.strip()]
+    for candidate in (text, lines[-1] if lines else ""):
+        cleaned = candidate.strip(_WRAPPERS)
+        if cleaned.casefold().startswith("intent:"):
+            cleaned = cleaned[len("intent:") :].strip(_WRAPPERS)
+        label = labels.get(cleaned.casefold())
+        if label:
+            return label
+    return None
+
+
+def macro_f1(pairs: list[tuple[str, str | None]]) -> float:
+    """Unweighted mean F1 over every intent that is gold or predicted."""
+    true_pos: Counter[str] = Counter()
+    predicted: Counter[str] = Counter()
+    gold: Counter[str] = Counter()
+    for expected, guess in pairs:
+        gold[expected] += 1
+        if guess is not None:
+            predicted[guess] += 1
+            true_pos[guess] += guess == expected
+    labels = set(gold) | set(predicted)
+    if not labels:
+        return 0.0
+    scores = [
+        2 * true_pos[label] / (gold[label] + predicted[label]) for label in labels
+    ]
+    return sum(scores) / len(scores)
+
+
+class Banking77:
+    name = "banking77"
+    decision_instructions = "Which intent does the customer's message express?"
+
+    def load_tasks(self) -> list[Task]:
+        tasks = []
+        with open(DATASET, encoding="utf-8") as f:
+            for line in f:
+                d = json.loads(line)
+                tasks.append(
+                    Task(
+                        id=d["task_id"],
+                        prompt=d["question"],
+                        metadata={"answer": d["answer"]},
+                    )
+                )
+        return tasks
+
+    def build_prompt(self, task: Task) -> str:
+        listing = "\n".join(f"- {label}" for label in intents())
+        return f"{SYSTEM}\n\nIntents:\n{listing}\n\nMessage: {task.prompt}"
+
+    def decision_criteria(self, task: Task) -> dict[str, str]:
+        """All 77 intents, keyed by label so the chosen key is the answer."""
+        return {label: label.replace("_", " ").lower() for label in intents()}
+
+    def evaluate_with_feedback(self, task: Task, response: str) -> EvaluationResult:
+        predicted = extract_intent(response)
+        expected = task.metadata["answer"]
+        if predicted is None:
+            feedback = (
+                "The answer did not name one of the listed intents. Reply with "
+                "exactly one intent name from the list and nothing else."
+            )
+        elif predicted != expected:
+            feedback = (
+                f"{predicted} is not the intent of this message. Re-read it and "
+                "reply with exactly one intent name from the list."
+            )
+        else:
+            feedback = ""
+        return EvaluationResult(
+            score=float(predicted == expected),
+            feedback=feedback,
+            details={"predicted_intent": predicted, "expected_intent": expected},
+        )
+
+    def evaluate(self, task: Task, response: str) -> bool:
+        return extract_intent(response) == task.metadata["answer"]
+
+    def summary_fields(self, records: list[object]) -> dict:
+        """Macro-F1 over the scored tasks, for a router's per-intent quality.
+
+        Tasks that never reached the verifier (timeouts, loop kills) count as
+        no prediction, exactly as they count against accuracy.
+        """
+        gold = _gold()
+        pairs = [
+            (
+                gold[record.task_id],
+                (getattr(record, "workspace", None) or {}).get("predicted_intent"),
+            )
+            for record in records
+            if record.task_id in gold
+        ]
+        return {"macro_f1": round(macro_f1(pairs) * 100, 1)} if pairs else {}

benchkit/benchmarks/base.py

@@ -0,0 +1,19 @@
+"""Benchmark base protocol."""
+
+from dataclasses import dataclass, field
+from typing import Protocol
+
+
+@dataclass
+class Task:
+    id: str
+    prompt: str
+    metadata: dict = field(default_factory=dict)
+
+
+class Benchmark(Protocol):
+    name: str
+
+    def load_tasks(self) -> list[Task]: ...
+    def build_prompt(self, task: Task) -> str: ...
+    def evaluate(self, task: Task, response: str) -> bool | float: ...

benchkit/benchmarks/boolq.py

@@ -0,0 +1,49 @@
+"""BoolQ benchmark - yes/no reading comprehension."""
+
+import json
+import re
+from pathlib import Path
+
+from benchkit.benchmarks.base import Task
+from benchkit.benchmarks.utils import strip_think_tags
+
+DATASET = Path(__file__).parent.parent / "datasets" / "boolq.jsonl"
+
+SYSTEM = "Read the passage and answer the question. Reply with ONLY YES or NO."
+
+
+def _extract_answer(response: str) -> str | None:
+    text = strip_think_tags(response)
+    answers = re.findall(r"\b(?:yes|no)\b", text.lower())
+    return answers[-1] if answers else None
+
+
+class BoolQ:
+    name = "boolq"
+    # Yes/no without a choices list; two options means no repair.
+    answer_options = 2
+    decision_instructions = "Based on the passage, is the answer to the question yes?"
+
+    def load_tasks(self) -> list[Task]:
+        tasks = []
+        with open(DATASET) as f:
+            for line in f:
+                d = json.loads(line)
+                tasks.append(
+                    Task(
+                        id=d["task_id"],
+                        prompt=d["question"],
+                        metadata={
+                            "passage": d["passage"],
+                            "answer": d["answer"],
+                        },
+                    )
+                )
+        return tasks
+
+    def build_prompt(self, task: Task) -> str:
+        passage = task.metadata["passage"]
+        return f"{SYSTEM}\n\nPassage:\n{passage}\n\nQuestion: {task.prompt}"
+
+    def evaluate(self, task: Task, response: str) -> bool:
+        return _extract_answer(response) == task.metadata["answer"]