benchkit-cli 0.1.0__py3-none-any.whl

benchkit/benchmarks/mmvp.py

@@ -0,0 +1,115 @@
+"""MMVP - 150 pairs of look-alike images that CLIP-style encoders confuse.
+
+Each pair asks the same two-option question about two images that differ in
+one visual detail (orientation, count, state, colour), and the right answer
+differs between them. The official score is pair accuracy: both questions of
+a pair must be right, so a model that ignores the image sits at 25%, not 50%.
+Plain per-question accuracy is the headline score so BenchKit's averages and
+decision calibration stay comparable; pair accuracy is reported next to it.
+Pairs are consecutive, so even-sized slices keep them whole.
+"""
+
+import csv
+import io
+import re
+from functools import cache
+
+from benchkit.benchmarks.base import Task
+from benchkit.benchmarks.mcq import extract_choice
+from benchkit.benchmarks.utils import strip_think_tags
+from benchkit.benchmarks.vision import load_files, vision_dataset
+
+DATASET = "MMVP/MMVP"
+MANIFEST = "Questions.csv"
+REVISION = "37eafecab8a3940c50c2ade5b36de69dbc99a8cf"
+
+SYSTEM = "Answer the question about the image. Reply with ONLY the letter (A or B)."
+
+_OPTIONS = re.compile(r"\(a\)\s*(.+?)\s*\(b\)\s*(.+)", re.IGNORECASE)
+
+
+def parse_manifest(data: bytes) -> list[dict]:
+    rows = []
+    for row in csv.DictReader(io.StringIO(data.decode("utf-8-sig"))):
+        index = int(row["Index"])
+        options = _OPTIONS.fullmatch(row["Options"].strip())
+        if options is None:
+            raise ValueError(f"MMVP row {index}: unexpected options {row['Options']}")
+        rows.append(
+            {
+                "task_id": f"mmvp/{index}",
+                "pair": (index + 1) // 2,
+                "question": row["Question"].strip(),
+                "choices": [options.group(1).strip(), options.group(2).strip()],
+                "answer": row["Correct Answer"].strip().strip("()").upper(),
+                "image": f"MMVP Images/{index}.jpg",
+            }
+        )
+    return rows
+
+
+@cache
+def _rows() -> tuple[dict, ...]:
+    return tuple(load_files("mmvp", DATASET, MANIFEST, REVISION, parse_manifest))
+
+
+def pair_accuracy(records: list[object]) -> dict:
+    """Share of pairs with both questions right, over pairs fully scored."""
+    pair_of = {row["task_id"]: row["pair"] for row in _rows()}
+    pairs: dict[int, list[float]] = {}
+    for record in records:
+        pair = pair_of.get(record.task_id)
+        if pair is not None:
+            pairs.setdefault(pair, []).append(float(getattr(record, "score", 0.0)))
+    complete = [scores for scores in pairs.values() if len(scores) == 2]
+    if not complete:
+        return {}
+    passed = sum(all(score >= 1.0 for score in scores) for scores in complete)
+    return {
+        "pair_accuracy": round(100 * passed / len(complete), 1),
+        "pairs": len(complete),
+    }
+
+
+class MMVP:
+    name = "mmvp"
+    vision = True
+    task_count = 300
+    data_source = vision_dataset(
+        "mmvp", REVISION, "MMVP image pairs from Hugging Face", _rows
+    )
+    decision_instructions = (
+        "Which option correctly answers the question about the image?"
+    )
+
+    def load_tasks(self) -> list[Task]:
+        return [
+            Task(
+                id=row["task_id"],
+                prompt=row["question"],
+                metadata={
+                    "choices": row["choices"],
+                    "answer": row["answer"],
+                    "images": [row["image"]],
+                },
+            )
+            for row in _rows()
+        ]
+
+    def build_prompt(self, task: Task) -> str:
+        choices = "\n".join(
+            f"{letter}) {choice}"
+            for letter, choice in zip("AB", task.metadata["choices"], strict=True)
+        )
+        return f"{SYSTEM}\n\n{task.prompt}\n\n{choices}"
+
+    def result_metadata(self, variant: str | None = None) -> dict:
+        return {"dataset_revision": REVISION}
+
+    def evaluate(self, task: Task, response: str) -> bool:
+        text = strip_think_tags(response)
+        choice = extract_choice(text, task.metadata["choices"], letters="AB")
+        return choice == task.metadata["answer"]
+
+    def summary_fields(self, records: list[object]) -> dict:
+        return pair_accuracy(records)

benchkit/benchmarks/ocrbench.py

@@ -0,0 +1,96 @@
+"""OCRBench - text recognition and text-centric VQA over 1,000 images.
+
+Ten categories from scene text to handwritten math. Scoring follows the
+official OCRBench evaluator: a task passes when any reference answer appears
+in the response, case-insensitively. Handwritten math (HME100k) compares with
+all whitespace removed and keeps case, since LaTeX is case-sensitive. The
+question is sent as written, with no extra instructions, as upstream does.
+"""
+
+from functools import cache
+
+from benchkit.benchmarks.base import Task
+from benchkit.benchmarks.utils import strip_think_tags
+from benchkit.benchmarks.vision import (
+    category_scores,
+    load_rows,
+    spread,
+    vision_dataset,
+)
+from benchkit.evaluation import EvaluationResult
+
+DATASET = "echo840/OCRBench"
+PARQUET = "data/test-00000-of-00001.parquet"
+REVISION = "92a54bd1384387c178d5a07140a2d85e0a3d12e1"
+
+
+def _convert(index: int, row: dict) -> dict:
+    return {
+        "task_id": f"ocrbench/{index}",
+        "source": row["dataset"],
+        "category": row["question_type"],
+        "question": row["question"],
+        "answers": list(row["answer"]),
+    }
+
+
+def matches(response: str, answers: list[str], source: str) -> bool:
+    predicted = strip_think_tags(response).strip().replace("\n", " ")
+    if source == "HME100k":
+        predicted = predicted.replace(" ", "")
+        return any(answer.strip().replace(" ", "") in predicted for answer in answers)
+    predicted = predicted.lower()
+    return any(
+        answer.lower().strip().replace("\n", " ") in predicted for answer in answers
+    )
+
+
+@cache
+def _rows() -> tuple[dict, ...]:
+    return tuple(load_rows("ocrbench", DATASET, PARQUET, REVISION, _convert))
+
+
+class OCRBench:
+    name = "ocrbench"
+    vision = True
+    task_count = 1000
+    data_source = vision_dataset(
+        "ocrbench", REVISION, "OCRBench images from Hugging Face", _rows
+    )
+
+    def load_tasks(self) -> list[Task]:
+        tasks = [
+            Task(
+                id=row["task_id"],
+                prompt=row["question"],
+                metadata={
+                    "answers": row["answers"],
+                    "source": row["source"],
+                    "category": row["category"],
+                    "images": [row["image"]],
+                },
+            )
+            for row in _rows()
+        ]
+        return spread(tasks, lambda task: task.metadata["category"])
+
+    def build_prompt(self, task: Task) -> str:
+        return task.prompt
+
+    def result_metadata(self, variant: str | None = None) -> dict:
+        return {"dataset_revision": REVISION}
+
+    def evaluate_with_feedback(self, task: Task, response: str) -> EvaluationResult:
+        passed = matches(response, task.metadata["answers"], task.metadata["source"])
+        return EvaluationResult(
+            score=float(passed),
+            feedback="" if passed else "The answer does not match the image text.",
+        )
+
+    def evaluate(self, task: Task, response: str) -> bool:
+        return matches(response, task.metadata["answers"], task.metadata["source"])
+
+    def summary_fields(self, records: list[object]) -> dict:
+        categories = {row["task_id"]: row["category"] for row in _rows()}
+        scores = category_scores(records, categories)
+        return {"category_scores": scores} if scores else {}

benchkit/benchmarks/openbookqa.py

@@ -0,0 +1,52 @@
+"""OpenBookQA benchmark - elementary science multiple choice reasoning."""
+
+import json
+from pathlib import Path
+
+from benchkit.benchmarks.base import Task
+from benchkit.benchmarks.mcq import extract_choice
+from benchkit.benchmarks.utils import strip_think_tags
+
+DATASET = Path(__file__).parent.parent / "datasets" / "openbookqa.jsonl"
+
+SYSTEM = (
+    "Answer the following multiple choice science question. "
+    "Reply with ONLY the letter (A, B, C, or D)."
+)
+
+
+def _format_choices(choices: list[str]) -> str:
+    return "\n".join(
+        f"{letter}) {choice}" for letter, choice in zip("ABCD", choices, strict=False)
+    )
+
+
+class OpenBookQA:
+    name = "openbookqa"
+    decision_instructions = "Which option correctly answers this science question?"
+
+    def load_tasks(self) -> list[Task]:
+        tasks = []
+        with open(DATASET) as f:
+            for line in f:
+                d = json.loads(line)
+                tasks.append(
+                    Task(
+                        id=d["task_id"],
+                        prompt=d["question"],
+                        metadata={
+                            "choices": d["choices"],
+                            "answer": d["answer"],
+                        },
+                    )
+                )
+        return tasks
+
+    def build_prompt(self, task: Task) -> str:
+        choices = _format_choices(task.metadata["choices"])
+        return f"{SYSTEM}\n\n{task.prompt}\n\n{choices}"
+
+    def evaluate(self, task: Task, response: str) -> bool:
+        text = strip_think_tags(response)
+        choice = extract_choice(text, task.metadata["choices"])
+        return choice == task.metadata["answer"]