benchkit-cli 0.1.0__py3-none-any.whl

benchkit/benchmarks/chartqa.py

@@ -0,0 +1,133 @@
+"""ChartQA - questions about bar, line and pie charts (2,500 test images).
+
+Half the questions are human-written and need reasoning across the chart;
+half are machine-generated lookups. Scoring is the standard relaxed accuracy:
+a numeric answer passes within 5% of the reference, anything else must match
+exactly up to case. Slices interleave the two halves.
+
+One deliberate leniency over the reference scorer: no test label carries a
+"%", yet chat models write "6.8%" for a label of 6.8. A percent answer is
+therefore read both as written and divided by 100, and passes if either fits.
+"""
+
+import re
+from functools import cache
+
+from benchkit.benchmarks.base import Task
+from benchkit.benchmarks.utils import strip_think_tags
+from benchkit.benchmarks.vision import (
+    category_scores,
+    load_rows,
+    spread,
+    vision_dataset,
+)
+from benchkit.evaluation import EvaluationResult
+
+DATASET = "HuggingFaceM4/ChartQA"
+PARQUET = "data/test-00000-of-00001-e2cd0b7a0f9eb20d.parquet"
+REVISION = "b605b6e08b57faf4359aeb2fe6a3ca595f99b6c5"
+
+INSTRUCTION = (
+    "Answer the question about the chart. You may reason first, but end with "
+    'a final line of the form "Answer: X", where X is a single word or a '
+    "number without units."
+)
+
+_ANSWER = re.compile(r"answer\s*[:：]\s*(.+)", re.IGNORECASE)
+_WRAPPERS = " \t`'\"*_$"
+
+
+def _convert(index: int, row: dict) -> dict:
+    return {
+        "task_id": f"chartqa/{index}",
+        "category": "augmented" if row["human_or_machine"] else "human",
+        "question": row["query"],
+        "answers": list(row["label"]),
+    }
+
+
+def extract_answer(response: str) -> str:
+    """The text after the last "Answer:", else the last non-empty line."""
+    text = strip_think_tags(response).strip()
+    found = _ANSWER.findall(text)
+    if found:
+        answer = found[-1]
+    else:
+        lines = [line for line in text.splitlines() if line.strip()]
+        answer = lines[-1] if lines else ""
+    return answer.strip(_WRAPPERS).removesuffix(".").strip(_WRAPPERS)
+
+
+def _to_floats(text: str) -> list[float]:
+    """Numeric readings of an answer: a percent is tried over 100 and raw."""
+    text = text.replace(",", "").strip()
+    try:
+        value = float(text.removesuffix("%").strip())
+    except ValueError:
+        return []
+    return [value / 100.0, value] if text.endswith("%") else [value]
+
+
+def relaxed_match(prediction: str, target: str, tolerance: float = 0.05) -> bool:
+    expected = _to_floats(target)
+    predicted = _to_floats(prediction)
+    if predicted and expected and expected[0]:
+        return any(
+            abs(value - expected[0]) / abs(expected[0]) <= tolerance
+            for value in predicted
+        )
+    return prediction.strip().lower() == target.strip().lower()
+
+
+@cache
+def _rows() -> tuple[dict, ...]:
+    return tuple(load_rows("chartqa", DATASET, PARQUET, REVISION, _convert))
+
+
+class ChartQA:
+    name = "chartqa"
+    vision = True
+    task_count = 2500
+    data_source = vision_dataset(
+        "chartqa", REVISION, "ChartQA test charts from Hugging Face", _rows
+    )
+
+    def load_tasks(self) -> list[Task]:
+        tasks = [
+            Task(
+                id=row["task_id"],
+                prompt=row["question"],
+                metadata={
+                    "answers": row["answers"],
+                    "category": row["category"],
+                    "images": [row["image"]],
+                },
+            )
+            for row in _rows()
+        ]
+        return spread(tasks, lambda task: task.metadata["category"])
+
+    def build_prompt(self, task: Task) -> str:
+        return f"{INSTRUCTION}\n\nQuestion: {task.prompt}"
+
+    def result_metadata(self, variant: str | None = None) -> dict:
+        return {"dataset_revision": REVISION}
+
+    def evaluate_with_feedback(self, task: Task, response: str) -> EvaluationResult:
+        predicted = extract_answer(response)
+        passed = any(relaxed_match(predicted, a) for a in task.metadata["answers"])
+        return EvaluationResult(
+            score=float(passed),
+            feedback=""
+            if passed
+            else 'The final answer is wrong. End with one line "Answer: X".',
+            details={"predicted_answer": predicted},
+        )
+
+    def evaluate(self, task: Task, response: str) -> bool:
+        return self.evaluate_with_feedback(task, response).passed
+
+    def summary_fields(self, records: list[object]) -> dict:
+        categories = {row["task_id"]: row["category"] for row in _rows()}
+        scores = category_scores(records, categories)
+        return {"category_scores": scores} if scores else {}

benchkit/benchmarks/code_verifier.py

@@ -0,0 +1,77 @@
+"""Code verifier - does a model-written HumanEval+ solution pass its tests?"""
+
+import json
+import re
+from pathlib import Path
+
+from benchkit.benchmarks.base import Task
+from benchkit.benchmarks.utils import strip_think_tags
+
+DATASET = Path(__file__).parent.parent / "datasets" / "code_verifier.jsonl"
+
+# One neutral question for both harnesses, so neither is nudged either way.
+QUESTION = "Does this solution correctly solve the problem?"
+
+
+def _extract_answer(response: str) -> str | None:
+    text = strip_think_tags(response)
+    answers = re.findall(r"\b(?:yes|no)\b", text.lower())
+    return answers[-1] if answers else None
+
+
+class CodeVerifier:
+    name = "code-verifier"
+    # Yes/no without a choices list; two options means no repair.
+    answer_options = 2
+    decision_instructions = QUESTION
+    task_count = 500
+
+    def load_tasks(self) -> list[Task]:
+        tasks = []
+        with open(DATASET, encoding="utf-8") as f:
+            for line in f:
+                d = json.loads(line)
+                tasks.append(
+                    Task(
+                        id=d["id"],
+                        prompt=(
+                            f"Problem:\n```python\n{d['prompt'].strip()}\n```\n\n"
+                            f"Solution:\n```python\n"
+                            f"{d['solution'].strip()}\n```"
+                        ),
+                        metadata={
+                            "answer": "yes" if d["passes"] else "no",
+                            "task_id": d["task_id"],
+                            "model": d["model"],
+                        },
+                    )
+                )
+        return tasks
+
+    def build_prompt(self, task: Task) -> str:
+        return f"{task.prompt}\n\n{QUESTION} Reply with only YES or NO."
+
+    def evaluate(self, task: Task, response: str) -> bool:
+        return _extract_answer(response) == task.metadata["answer"]
+
+    def summary_fields(self, records: list[object]) -> dict:
+        """Score passing and failing solutions separately.
+
+        The suite is balanced, so a verifier that always says "yes" scores
+        50% overall; the per-class split shows which way a model leans.
+        """
+        answers = {task.id: task.metadata["answer"] for task in self.load_tasks()}
+        passing = [r for r in records if answers.get(r.task_id) == "yes"]
+        failing = [r for r in records if answers.get(r.task_id) == "no"]
+
+        def accuracy(rows: list[object]) -> float | None:
+            if not rows:
+                return None
+            return round(sum(r.passed for r in rows) / len(rows) * 100, 1)
+
+        return {
+            "verifier_pass_total": len(passing),
+            "verifier_fail_total": len(failing),
+            "verifier_pass_accuracy": accuracy(passing),
+            "verifier_fail_accuracy": accuracy(failing),
+        }

benchkit/benchmarks/downloads.py

@@ -0,0 +1,166 @@
+"""Datasets that are fetched on demand instead of shipping in the wheel.
+
+Small suites are bundled so ``--demo`` and smoke runs work offline. Anything
+large lives in a cache and is described by a :class:`Dataset` on the benchmark
+class, which lets ``benchkit datasets`` and the TUI list what is installed and
+install what is missing without loading any tasks.
+
+Plain files such as ``mmlu.jsonl`` are :class:`RemoteFile` downloads: a pinned
+URL plus a sha256, verified before the file is moved into place, so a partial
+or tampered download never reaches a run. They are cached under
+``~/.cache/benchkit/datasets`` (override with ``BENCHKIT_DATASET_CACHE``).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import threading
+import time
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path
+
+import httpx
+
+Progress = Callable[[int, int | None], None]
+"""Called with ``(bytes_done, bytes_total)`` while a download streams."""
+
+_ATTEMPTS = 4
+_LOCK = threading.Lock()
+# Pinned commit of this repository that still carried the bundled copies. A
+# commit pin plus a checksum keeps every install byte-identical to the file
+# earlier results were scored against.
+_PIN = "5789a5e2dcbb056abcbf32f810a1ed01de1cdad6"
+_RAW = f"https://raw.githubusercontent.com/DogukanUrker/BenchKit/{_PIN}"
+
+
+def cache_root() -> Path:
+    configured = os.environ.get("BENCHKIT_DATASET_CACHE")
+    if configured:
+        return Path(configured).expanduser()
+    return Path.home() / ".cache" / "benchkit" / "datasets"
+
+
+def fmt_bytes(size: float) -> str:
+    """Decimal units, matching rich's download progress column."""
+    for unit in ("B", "KB", "MB", "GB"):
+        if size < 1000 or unit == "GB":
+            return f"{size:.0f} {unit}" if unit == "B" else f"{size:.1f} {unit}"
+        size /= 1000
+    raise AssertionError("unreachable")
+
+
+@dataclass(frozen=True)
+class Dataset:
+    """What a benchmark needs on disk, and how to get it.
+
+    ``install`` is ``None`` for data BenchKit cannot fetch itself; ``hint``
+    then says how to provide it. ``size`` is the download size in bytes when it
+    is known (``approximate`` when it can drift, like a git clone); once the
+    data is installed, listings measure what is actually on disk instead.
+    """
+
+    summary: str
+    size: int | None
+    location: Callable[[], Path]
+    is_installed: Callable[[], bool]
+    install: Callable[[Progress | None], object] | None
+    hint: str = ""
+    approximate: bool = False
+
+
+@dataclass(frozen=True)
+class RemoteFile:
+    """One checksummed file fetched into the dataset cache on first use."""
+
+    name: str
+    url: str
+    sha256: str
+    size: int
+
+    @property
+    def path(self) -> Path:
+        return cache_root() / self.name
+
+    def installed(self) -> bool:
+        path = self.path
+        return path.is_file() and path.stat().st_size == self.size
+
+    def ensure(self, progress: Progress | None = None) -> Path:
+        """Return the cached path, downloading and verifying it if needed."""
+        with _LOCK:
+            if not self.installed():
+                self._download(progress)
+        return self.path
+
+    def source(self, summary: str) -> Dataset:
+        return Dataset(
+            summary=summary,
+            size=self.size,
+            location=lambda: self.path,
+            is_installed=self.installed,
+            install=self.ensure,
+        )
+
+    def _download(self, progress: Progress | None) -> None:
+        target = self.path
+        target.parent.mkdir(parents=True, exist_ok=True)
+        partial = target.with_name(target.name + ".part")
+        for attempt in range(_ATTEMPTS):
+            try:
+                digest = hashlib.sha256()
+                done = 0
+                with httpx.stream(
+                    "GET", self.url, follow_redirects=True, timeout=60
+                ) as r:
+                    r.raise_for_status()
+                    with open(partial, "wb") as f:
+                        for chunk in r.iter_bytes():
+                            f.write(chunk)
+                            digest.update(chunk)
+                            done += len(chunk)
+                            if progress:
+                                progress(done, self.size)
+                break
+            except (httpx.TransportError, httpx.HTTPStatusError) as exc:
+                retryable = not isinstance(exc, httpx.HTTPStatusError) or (
+                    exc.response.status_code in {429, 500, 502, 503, 504}
+                )
+                if not retryable or attempt == _ATTEMPTS - 1:
+                    partial.unlink(missing_ok=True)
+                    raise DatasetUnavailable(self, str(exc)) from exc
+                time.sleep(2**attempt)
+
+        if digest.hexdigest() != self.sha256:
+            partial.unlink(missing_ok=True)
+            raise DatasetUnavailable(
+                self, f"checksum mismatch (got {digest.hexdigest()[:12]}…)"
+            )
+        partial.replace(target)
+
+
+class DatasetUnavailable(RuntimeError):
+    """A dataset is not cached and could not be fetched."""
+
+    def __init__(self, remote: RemoteFile, reason: str) -> None:
+        super().__init__(
+            f"{remote.name} is not installed and could not be downloaded "
+            f"({reason}). Run `benchkit datasets install` once you are online, "
+            f"or place the file at {remote.path} (sha256 {remote.sha256})."
+        )
+
+
+MMLU = RemoteFile(
+    name="mmlu.jsonl",
+    url=f"{_RAW}/src/benchkit/datasets/mmlu.jsonl",
+    sha256="e3e9beb9b3f6b342b99ef59ead76a03c7553fb86705ce17ce95e58e51c1cc37d",
+    size=8_563_175,
+)
+
+MMLU_PRO = RemoteFile(
+    name="mmlu_pro.jsonl",
+    url=f"{_RAW}/src/benchkit/datasets/mmlu_pro.jsonl",
+    sha256="92ec532d9e2bae342ec7256005c20b6e5defbed76504280df775fb77068f077e",
+    size=10_260_787,
+)

benchkit/benchmarks/evalplus.py

@@ -0,0 +1,288 @@
+"""EvalPlus benchmarks with stronger HumanEval+ and MBPP+ test suites."""
+
+from __future__ import annotations
+
+import contextlib
+import multiprocessing
+import os
+import platform
+import textwrap
+from functools import cache, lru_cache
+from pathlib import Path
+from typing import Any
+
+from benchkit.benchmarks.base import Task
+from benchkit.benchmarks.downloads import Dataset
+from benchkit.benchmarks.utils import strip_think_tags
+
+HUMANEVAL_SYSTEM = (
+    "Complete the following Python function. "
+    "Output ONLY the function body. "
+    "No markdown fences, no explanation, no examples."
+)
+MBPP_SYSTEM = (
+    "Write a Python function to solve the following task. "
+    "Output ONLY the function. "
+    "No markdown fences, no explanation."
+)
+
+
+@contextlib.contextmanager
+def _fork_multiprocessing():
+    """Temporarily swap ``multiprocessing`` primitives to use the *fork* context.
+
+    EvalPlus calls ``multiprocessing.Process`` (and friends) using the
+    module-level default, which on Python 3.13 / macOS is ``spawn``.  The
+    ``spawn`` path validates every FD it intends to keep open across
+    ``fork_exec``, raising ``ValueError: bad value(s) in fds_to_keep`` when
+    the Textual TUI has high-numbered or already-closed descriptors.
+
+    Switching to ``fork`` side-steps the issue entirely: the child inherits a
+    copy of the parent's address space and no FDs need to be explicitly
+    forwarded.  The swap is scoped so the rest of the process is unaffected.
+
+    EvalPlus's ``evalplus.eval`` does ``from multiprocessing import Array, Value``
+    so those names are bound on that module and must be patched there as well.
+    """
+    import evalplus.eval as _ev
+
+    fork_ctx = multiprocessing.get_context("fork")
+    mp_originals = (
+        multiprocessing.Process,
+        multiprocessing.Value,
+        multiprocessing.Array,
+    )
+    ev_originals = (_ev.Value, _ev.Array)
+    try:
+        multiprocessing.Process = fork_ctx.Process
+        multiprocessing.Value = fork_ctx.Value
+        multiprocessing.Array = fork_ctx.Array
+        _ev.Value = fork_ctx.Value
+        _ev.Array = fork_ctx.Array
+        yield
+    finally:
+        multiprocessing.Process, multiprocessing.Value, multiprocessing.Array = (
+            mp_originals
+        )
+        _ev.Value, _ev.Array = ev_originals
+
+
+def _evalplus_api():
+    """Import EvalPlus lazily so normal startup stays lightweight."""
+    # RLIMIT_AS is not reliably supported on macOS. EvalPlus otherwise tries
+    # to impose a 4 GiB address-space limit that can make every check fail
+    # before generated code starts.
+    if platform.system() == "Darwin":
+        os.environ.setdefault("EVALPLUS_MAX_MEMORY_BYTES", "-1")
+
+    try:
+        from evalplus.data import get_human_eval_plus, get_mbpp_plus
+        from evalplus.eval import PASS
+        from evalplus.eval._special_oracle import MBPP_OUTPUT_NOT_NONE_TASKS
+        from evalplus.evaluate import check_correctness
+        from evalplus.gen.util import trusted_exec
+    except ImportError as exc:
+        raise RuntimeError(
+            "EvalPlus is missing from the environment. Run: uv sync"
+        ) from exc
+
+    return (
+        get_human_eval_plus,
+        get_mbpp_plus,
+        PASS,
+        MBPP_OUTPUT_NOT_NONE_TASKS,
+        check_correctness,
+        trusted_exec,
+    )
+
+
+@lru_cache(maxsize=2)
+def _problems(dataset: str) -> dict[str, dict[str, Any]]:
+    get_human_eval_plus, get_mbpp_plus, *_ = _evalplus_api()
+    if dataset == "humaneval":
+        return get_human_eval_plus()
+    if dataset == "mbpp":
+        return get_mbpp_plus()
+    raise ValueError(f"Unsupported EvalPlus dataset: {dataset}")
+
+
+@cache
+def _expected_output(dataset: str, task_id: str) -> dict[str, list[Any]]:
+    """Compute the official oracle outputs once per task and process."""
+    (
+        _,
+        _,
+        _,
+        output_not_none_tasks,
+        _,
+        trusted_exec,
+    ) = _evalplus_api()
+    problem = _problems(dataset)[task_id]
+    oracle_code = problem["prompt"] + problem["canonical_solution"]
+    output_not_none = (
+        dataset == "mbpp" and problem["entry_point"] in output_not_none_tasks
+    )
+
+    base, base_time = trusted_exec(
+        oracle_code,
+        problem["base_input"],
+        problem["entry_point"],
+        record_time=True,
+        output_not_none=output_not_none,
+    )
+    plus, plus_time = trusted_exec(
+        oracle_code,
+        problem["plus_input"],
+        problem["entry_point"],
+        record_time=True,
+        output_not_none=output_not_none,
+    )
+    return {
+        "base": base,
+        "base_time": base_time,
+        "plus": plus,
+        "plus_time": plus_time,
+    }
+
+
+def _extract_code(response: str) -> str:
+    """Strip reasoning and a single Markdown code fence."""
+    text = strip_think_tags(response).rstrip()
+    if "```python" in text:
+        text = text.split("```python", 1)[1].split("```", 1)[0]
+    elif "```" in text:
+        text = text.split("```", 1)[1].split("```", 1)[0]
+    return text.strip("\n")
+
+
+def _human_solution(problem: dict[str, Any], response: str) -> str:
+    code = _extract_code(response)
+    entry_point = problem["entry_point"]
+    if f"def {entry_point}" not in code:
+        normalized = textwrap.dedent(code)
+        solution = problem["prompt"] + textwrap.indent(normalized, "    ")
+        try:
+            compile(solution, "<benchkit-humaneval>", "exec")
+        except (IndentationError, TabError):
+            lines = code.splitlines()
+            first_index = next(
+                (index for index, line in enumerate(lines) if line.strip()), None
+            )
+            if first_index is not None and not lines[first_index].startswith(
+                (" ", "\t")
+            ):
+                lines[first_index] = "    " + lines[first_index]
+                normalized = textwrap.dedent("\n".join(lines))
+                repaired = problem["prompt"] + textwrap.indent(normalized, "    ")
+                try:
+                    compile(repaired, "<benchkit-humaneval>", "exec")
+                except (IndentationError, TabError):
+                    pass
+                else:
+                    solution = repaired
+        return solution
+
+    imports = [
+        line
+        for line in problem["prompt"].splitlines()
+        if line.startswith(("import ", "from "))
+    ]
+    import_block = "\n".join(imports)
+    return f"{import_block}\n\n{code}" if imports else code
+
+
+def _evalplus_cache(dataset: str) -> Path:
+    from evalplus.data.humaneval import HUMANEVAL_PLUS_VERSION
+    from evalplus.data.mbpp import MBPP_PLUS_VERSION
+    from evalplus.data.utils import get_dataset_metadata
+
+    if dataset == "humaneval":
+        name, version = "HumanEvalPlus", HUMANEVAL_PLUS_VERSION
+    else:
+        name, version = "MbppPlus", MBPP_PLUS_VERSION
+    return Path(get_dataset_metadata(name, version, mini=False)[1])
+
+
+def _evalplus_installed(dataset: str) -> bool:
+    try:
+        return _evalplus_cache(dataset).exists()
+    except ImportError:
+        return False
+
+
+def _evalplus_source(dataset: str, summary: str, size: int) -> Dataset:
+    return Dataset(
+        summary=summary,
+        size=size,
+        location=lambda: _evalplus_cache(dataset),
+        is_installed=lambda: _evalplus_installed(dataset),
+        install=lambda _progress: _quiet_problems(dataset),
+    )
+
+
+def _quiet_problems(dataset: str) -> None:
+    # EvalPlus downloads through wget, which draws its own progress bar on
+    # stdout and tears up the CLI's live progress line and the TUI.
+    with open(os.devnull, "w") as sink, contextlib.redirect_stdout(sink):
+        _problems(dataset)
+
+
+class _EvalPlusBenchmark:
+    dataset: str
+    name: str
+    task_count: int
+    system: str
+    evaluation_activity = "executing code and running EvalPlus tests"
+
+    def load_tasks(self) -> list[Task]:
+        return [
+            Task(
+                id=task_id,
+                prompt=problem["prompt"],
+                metadata={
+                    "entry_point": problem["entry_point"],
+                    "canonical_solution": problem["canonical_solution"],
+                },
+            )
+            for task_id, problem in _problems(self.dataset).items()
+        ]
+
+    def build_prompt(self, task: Task) -> str:
+        return f"{self.system}\n\n{task.prompt}"
+
+    def evaluate(self, task: Task, response: str) -> bool:
+        *_, pass_status, _, check_correctness, _ = _evalplus_api()
+        problem = _problems(self.dataset)[task.id]
+        if self.dataset == "humaneval":
+            solution = _human_solution(problem, response)
+        else:
+            solution = _extract_code(response)
+
+        with _fork_multiprocessing():
+            result = check_correctness(
+                self.dataset,
+                completion_id=0,
+                problem=problem,
+                solution=solution,
+                expected_output=_expected_output(self.dataset, task.id),
+                fast_check=True,
+            )
+        return result["base"][0] == pass_status and result["plus"][0] == pass_status
+
+
+class HumanEvalPlus(_EvalPlusBenchmark):
+    name = "humaneval-plus"
+    dataset = "humaneval"
+    task_count = 164
+    system = HUMANEVAL_SYSTEM
+    data_source = _evalplus_source(
+        "humaneval", "HumanEval+ release from EvalPlus", size=925_932
+    )
+
+
+class MBPPPlus(_EvalPlusBenchmark):
+    name = "mbpp-plus"
+    dataset = "mbpp"
+    task_count = 378
+    system = MBPP_SYSTEM
+    data_source = _evalplus_source("mbpp", "MBPP+ release from EvalPlus", size=336_032)