eval-framework 0.13.0__tar.gz → 0.13.2__tar.gz

{eval_framework-0.13.0 → eval_framework-0.13.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.13.0
+Version: 0.13.2
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -232,7 +232,7 @@ Requires-Dist: jsonlines>=4,<5
 Requires-Dist: lxml>=6.1.3,<7
 Requires-Dist: python-iso639>=2026.7.23
 Requires-Dist: wandb>=0.30.0,<1
-Requires-Dist: boto3>=1.43.93,<2
+Requires-Dist: boto3>=1.43.94,<2
 Requires-Dist: numpy>=2.5.3
 Requires-Dist: antlr4-python3-runtime==4.11.0
 Requires-Dist: scipy>=1.18.1,<2
@@ -241,7 +241,7 @@ Requires-Dist: eval-framework[determined,api,openai,transformers,accelerate,opti
 Requires-Dist: aleph-alpha-client>=11.5.1 ; extra == 'api'
 Requires-Dist: determined>=0.38.1,<0.39 ; extra == 'determined'
 Requires-Dist: tensorboard==2.21.0 ; extra == 'determined'
-Requires-Dist: openai>=3.13.0,<4 ; extra == 'openai'
+Requires-Dist: openai>=3.14.0,<4 ; extra == 'openai'
 Requires-Dist: tiktoken>=0.14.0,<1 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'optional'

{eval_framework-0.13.0 → eval_framework-0.13.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.13.0"
+version = "0.13.2"
 description = "Evaluation Framework"
 readme = "README.md"
 requires-python = ">=3.12,<3.15"
@@ -35,7 +35,7 @@ dependencies = [
     "lxml>=6.1.3,<7",
     "python-iso639>=2026.7.23",
     "wandb>=0.30.0,<1",
-    "boto3>=1.43.93,<2",
+    "boto3>=1.43.94,<2",
     "numpy>=2.5.3",
     "antlr4-python3-runtime==4.11.0",
     "scipy>=1.18.1,<2",
@@ -54,7 +54,7 @@ determined = [
 ]
 api = ["aleph-alpha-client>=11.5.1"]
 openai = [
-    "openai>=3.13.0,<4",
+    "openai>=3.14.0,<4",
     "tiktoken>=0.14.0,<1",
     "transformers>=4.45.2,<5",
 ]

{eval_framework-0.13.0 → eval_framework-0.13.2}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.13.0"
+version = "0.13.2"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -39,7 +39,7 @@ dependencies = [
   "lxml>=6.1.3,<7",
   "python-iso639>=2026.7.23",
   "wandb>=0.30.0,<1",
-  "boto3>=1.43.93,<2",
+  "boto3>=1.43.94,<2",
   "numpy>=2.5.3",
   # is a dependency of sympy, but not explicitly listed in the requirements.txt
   # https://github.com/sympy/sympy/blob/0204fa34e8f6f6f8ccb4de01209be9a2345c9d6e/doc/src/contributing/dependencies.md?plain=1#L125
@@ -55,7 +55,7 @@ determined = [
 ]
 api = ["aleph-alpha-client>=11.5.1"]
 openai = [
-  "openai>=3.13.0,<4",
+  "openai>=3.14.0,<4",
   "tiktoken>=0.14.0,<1",
   "transformers>=4.45.2,<5",
 ]

eval_framework-0.13.2/src/eval_framework/benchmarks/arc.py

@@ -0,0 +1,69 @@
+"""ARC (AI2 Reasoning Challenge): https://huggingface.co/datasets/allenai/ai2_arc
+
+Grade-school science multiple-choice questions, split into the ARC-Easy and ARC-Challenge subjects. The
+base task scores the full answer text (cloze); the OLMES variant shows the options as space-prefixed
+lettered choices and scores the letters; the IDK variant lets the model abstain with "I do not know".
+"""
+
+from typing import Any, final, override
+
+from eval_framework.choices import ChoiceFields, ChoiceReader
+from eval_framework.composed import ComposedBenchmark
+from eval_framework.contract import Benchmark
+from eval_framework.subjects import ListOfSubjects
+from eval_framework.tasks.base import Language
+from eval_framework.tasks.dataset_loading import DatasetPolicy
+from eval_framework.tasks.dataset_revisions import pinned_by_framework
+from eval_framework.tasks.task_style import ClozeStyle, MCStyle, TaskStyler, answer_key_to_index
+
+_IDK_PREAMBLE = (
+    "Answer only if you are confident, since mistakes may be penalised, while correct answers receive points. "
+    "It is acceptable to answer with 'I do not know' if you are unsure, and you will receive 0 points."
+)
+
+
+@final
+class ArcReader(ChoiceReader):
+    """Reads an ARC item: the question and its answer options, with the correct one at ``answerKey`` —
+    a letter (A–E) or a 1-based number, both normalised to a 0-based index."""
+
+    @override
+    def read(self, item: dict[str, Any]) -> ChoiceFields:
+        return ChoiceFields(
+            raw_question=item["question"],
+            choices=item["choices"]["text"],
+            correct_index=answer_key_to_index(item["answerKey"]),
+        )
+
+
+def _arc_benchmark(id: str, styler: TaskStyler, dataset: DatasetPolicy | None = None) -> Benchmark:
+    dataset_policy = dataset if dataset is not None else pinned_by_framework("allenai/ai2_arc")
+    return ComposedBenchmark.choice(
+        id=id,
+        reader=ArcReader(),
+        styler=styler,
+        sample_split="test",
+        fewshot_split="train",
+        subjects=ListOfSubjects(["ARC-Easy", "ARC-Challenge"]),
+        dataset_policy=dataset_policy,
+        language=Language.ENG,
+    )
+
+
+def arc(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _arc_benchmark("ARC", ClozeStyle(question_prefix="Question: ", cue_text="Answer:"), dataset)
+
+
+def arc_olmes(dataset: DatasetPolicy | None = None) -> Benchmark:
+    styler = MCStyle(question_prefix="Question: ", cue_text="Answer:", space_prefixed_labels=True)
+    return _arc_benchmark("ARC_OLMES", styler, dataset)
+
+
+def arc_idk(dataset: DatasetPolicy | None = None) -> Benchmark:
+    styler = ClozeStyle(
+        question_prefix="Question: ", cue_text="Answer:", initial_prompt=lambda _subject: _IDK_PREAMBLE
+    ).with_abstention_option(" I do not know.")
+    return _arc_benchmark("ARC_IDK", styler, dataset)
+
+
+ARC_BENCHMARKS: list[Benchmark] = [arc(), arc_olmes(), arc_idk()]

eval_framework-0.13.2/src/eval_framework/benchmarks/arc_ellamind.py

@@ -0,0 +1,71 @@
+"""German ARC (EllaMind): https://huggingface.co/datasets/ellamind/arc-multilingual
+
+Grade-school science questions in German. Every slice loads the single German config (``deu``); the
+ARC-Easy and ARC-Challenge subjects are the rows of that config tagged by the ``arc_config`` column. Cloze
+scores the full answer text, MC the letter labels, BPB only the ground-truth answer's bits-per-byte.
+"""
+
+from typing import Any, final, override
+
+from eval_framework.choices import ChoiceFields, ChoiceReader
+from eval_framework.composed import ComposedBenchmark
+from eval_framework.contract import Benchmark
+from eval_framework.subjects import ListOfSubjects
+from eval_framework.tasks.base import Language
+from eval_framework.tasks.dataset_loading import DatasetPolicy
+from eval_framework.tasks.dataset_revisions import pinned_by_framework
+from eval_framework.tasks.task_style import BPBStyle, ClozeStyle, MCStyle, TaskStyler, answer_key_to_index
+
+
+@final
+class ArcEllamindReader(ChoiceReader):
+    """Reads a German ARC item: the question and its answer options, with the correct one at ``answer_key``
+    (a letter A–E or a 1-based number)."""
+
+    @override
+    def read(self, item: dict[str, Any]) -> ChoiceFields:
+        return ChoiceFields(
+            raw_question=item["question"],
+            choices=item["choices"],
+            correct_index=answer_key_to_index(item["answer_key"]),
+        )
+
+
+def _arc_ellamind_benchmark(id: str, styler: TaskStyler, dataset: DatasetPolicy | None = None) -> Benchmark:
+    # ARC-Easy and ARC-Challenge share the single "deu" config, tagged by the ``arc_config`` column.
+    source = (
+        dataset
+        if dataset is not None
+        else pinned_by_framework("ellamind/arc-multilingual").subject_encoded_in_column(
+            config="deu", column="arc_config"
+        )
+    )
+    return ComposedBenchmark.choice(
+        id=id,
+        reader=ArcEllamindReader(),
+        styler=styler,
+        sample_split="test",
+        fewshot_split="test",
+        subjects=ListOfSubjects(["ARC-Easy", "ARC-Challenge"]),
+        dataset_policy=source,
+        language=Language.DEU,
+    )
+
+
+def arc_ellamind_cloze_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _arc_ellamind_benchmark("ARC_ELLAMIND_CLOZE_DE", ClozeStyle.for_language(Language.DEU), dataset)
+
+
+def arc_ellamind_mc_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _arc_ellamind_benchmark("ARC_ELLAMIND_MC_DE", MCStyle.for_language(Language.DEU), dataset)
+
+
+def arc_ellamind_bpb_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _arc_ellamind_benchmark("ARC_ELLAMIND_BPB_DE", BPBStyle.for_language(Language.DEU), dataset)
+
+
+ARC_ELLAMIND_BENCHMARKS: list[Benchmark] = [
+    arc_ellamind_cloze_de(),
+    arc_ellamind_mc_de(),
+    arc_ellamind_bpb_de(),
+]

{eval_framework-0.13.0 → eval_framework-0.13.2}/src/eval_framework/benchmarks/gpqa_ellamind.py

@@ -13,7 +13,7 @@ from eval_framework.composed import ComposedBenchmark
 from eval_framework.contract import Benchmark
 from eval_framework.subjects import ListOfSubjects
 from eval_framework.tasks.base import Language
-from eval_framework.tasks.dataset_loading import DatasetPolicy, Subset
+from eval_framework.tasks.dataset_loading import DatasetPolicy
 from eval_framework.tasks.dataset_revisions import pinned_by_framework
 from eval_framework.tasks.task_style import BPBStyle, ClozeStyle, MCStyle, TaskStyler, shuffle_correct_with_distractors
 
@@ -47,8 +47,15 @@ def _gpqa_ellamind_benchmark(id: str, styler: TaskStyler, dataset: DatasetPolicy
 
 
 def _gpqa_ellamind_diamond_benchmark(id: str, styler: TaskStyler, dataset: DatasetPolicy | None = None) -> Benchmark:
-    source = dataset if dataset is not None else pinned_by_framework("ellamind/gpqa-multilingual")
-    return _gpqa_ellamind_benchmark(id, styler, Subset(source, keep=lambda row: row["is_diamond"]))
+    # The diamond variants keep only the ``is_diamond`` rows of the full dataset.
+    source = (
+        dataset
+        if dataset is not None
+        else pinned_by_framework("ellamind/gpqa-multilingual").subset(
+            lambda row: row["is_diamond"], description="the diamond subset"
+        )
+    )
+    return _gpqa_ellamind_benchmark(id, styler, source)
 
 
 def gpqa_ellamind_mc_de(dataset: DatasetPolicy | None = None) -> Benchmark:

{eval_framework-0.13.0 → eval_framework-0.13.2}/src/eval_framework/benchmarks/hle_ellamind.py

@@ -13,7 +13,7 @@ from eval_framework.composed import ComposedBenchmark
 from eval_framework.contract import Benchmark
 from eval_framework.subjects import ListOfSubjects
 from eval_framework.tasks.base import Language
-from eval_framework.tasks.dataset_loading import DatasetPolicy, Subset
+from eval_framework.tasks.dataset_loading import DatasetPolicy
 from eval_framework.tasks.dataset_revisions import pinned_by_framework
 from eval_framework.tasks.task_style import BPBStyle, ClozeStyle, MCStyle, TaskStyler, shuffle_correct_with_distractors
 
@@ -47,8 +47,15 @@ def _hle_ellamind_benchmark(id: str, styler: TaskStyler, dataset: DatasetPolicy
 
 
 def _hle_ellamind_native_benchmark(id: str, styler: TaskStyler, dataset: DatasetPolicy | None = None) -> Benchmark:
-    source = dataset if dataset is not None else pinned_by_framework("ellamind/hle-multilingual")
-    return _hle_ellamind_benchmark(id, styler, Subset(source, keep=lambda row: row["answer_type"] == "multipleChoice"))
+    # The NATIVE variants keep only the items that are natively multiple-choice in the original benchmark.
+    source = (
+        dataset
+        if dataset is not None
+        else pinned_by_framework("ellamind/hle-multilingual").subset(
+            lambda row: row["answer_type"] == "multipleChoice", description="the natively multiple-choice items"
+        )
+    )
+    return _hle_ellamind_benchmark(id, styler, source)
 
 
 def hle_ellamind_mc_de(dataset: DatasetPolicy | None = None) -> Benchmark:

eval_framework-0.13.2/src/eval_framework/benchmarks/social_iqa.py

@@ -0,0 +1,151 @@
+"""Social IQa (English): https://huggingface.co/datasets/allenai/social_i_qa
+
+Commonsense reasoning about social situations: a context and question with three answer options
+(``answerA``/``answerB``/``answerC``) and a 1-indexed ``label`` marking the correct one. The registered
+variant is OLMES-style: options are shown as space-prefixed lettered choices (" A. …") and the model is
+scored over the letter labels.
+
+``allenai/social_i_qa`` ships only a (no-longer-supported) loading script on its main branch, so it cannot
+be loaded by a pinned commit under ``datasets`` >= 4. We load Hugging Face's auto-generated parquet branch
+instead, falling back to the original AI2 Mosaic source if that branch is unavailable.
+"""
+
+import json
+import os
+import zipfile
+from pathlib import Path
+from typing import Any, final, override
+from urllib.request import urlretrieve
+
+from datasets import Dataset, DatasetDict, DownloadConfig, load_dataset
+
+from eval_framework.choices import ChoiceFields, ChoiceReader
+from eval_framework.composed import ComposedBenchmark
+from eval_framework.contract import Benchmark
+from eval_framework.tasks.base import Language
+from eval_framework.tasks.dataset_loading import DatasetLoader, DatasetPolicy
+from eval_framework.tasks.task_style import MCStyle
+
+SOCIAL_I_QA_DATASET_PATH = "allenai/social_i_qa"
+# Hugging Face auto-generates a parquet copy of every dataset on this hidden branch; we load from it
+# because the dataset itself only ships a loading script (unsupported since datasets 4.x) on its main branch.
+_PARQUET_REVISION = "refs/convert/parquet"
+_SOURCE_ZIP_URL = "https://storage.googleapis.com/ai2-mosaic/public/socialiqa/socialiqa-train-dev.zip"
+_ZIP_SUBDIR = "socialiqa-train-dev"
+
+
+def _cache_dir() -> str:
+    return os.environ.get("HF_DATASET_CACHE_DIR", f"{Path.home()}/.cache/huggingface/datasets")
+
+
+def _load_from_ai2_source(cache_dir: str) -> DatasetDict:
+    """Rebuild train/validation from AI2 Mosaic's original zip (jsonl rows paired with label files),
+    used only if the Hugging Face parquet branch cannot be reached."""
+    root = Path(cache_dir) / "social_i_qa_direct"
+    extract_dir = root / _ZIP_SUBDIR
+    if not extract_dir.exists() or not list(extract_dir.glob("*.jsonl")):
+        zip_path = root / "socialiqa-train-dev.zip"
+        zip_path.parent.mkdir(parents=True, exist_ok=True)
+        if not zip_path.exists():
+            urlretrieve(_SOURCE_ZIP_URL, zip_path)
+        with zipfile.ZipFile(zip_path, "r") as zf:
+            zf.extractall(zip_path.parent)
+
+    def read_split(jsonl_path: Path, labels_path: Path) -> list[dict[str, Any]]:
+        labels = labels_path.read_text(encoding="utf-8").splitlines()
+        rows = []
+        for idx, line in enumerate(jsonl_path.read_text(encoding="utf-8").splitlines()):
+            data = json.loads(line)
+            rows.append(
+                {
+                    "context": data["context"],
+                    "question": data["question"],
+                    "answerA": data["answerA"],
+                    "answerB": data["answerB"],
+                    "answerC": data["answerC"],
+                    "label": labels[idx].strip(),
+                }
+            )
+        return rows
+
+    return DatasetDict(
+        {
+            "train": Dataset.from_list(read_split(extract_dir / "train.jsonl", extract_dir / "train-labels.lst")),
+            "validation": Dataset.from_list(read_split(extract_dir / "dev.jsonl", extract_dir / "dev-labels.lst")),
+        }
+    )
+
+
+@final
+class _SocialIqaLoader(DatasetLoader):
+    """Loads social_i_qa from Hugging Face's parquet branch, falling back to the original AI2 source."""
+
+    def __init__(self, revision: str) -> None:
+        self._revision = revision
+
+    @override
+    def load(self, name: str | None) -> DatasetDict:
+        cache_dir = _cache_dir()
+        try:
+            return load_dataset(
+                SOCIAL_I_QA_DATASET_PATH,
+                revision=self._revision,
+                cache_dir=cache_dir,
+                download_config=DownloadConfig(cache_dir=cache_dir, max_retries=5),
+            )
+        except Exception:
+            return _load_from_ai2_source(cache_dir)
+
+    @override
+    def metadata(self) -> dict[str, str]:
+        return {"dataset_path": SOCIAL_I_QA_DATASET_PATH}
+
+
+@final
+class _SocialIqaDataset(DatasetPolicy):
+    """social_i_qa's data policy — see the module docstring for why it can't use a plain ``Pinned``."""
+
+    @override
+    def loader(self, custom_hf_revision: str | None) -> DatasetLoader:
+        return _SocialIqaLoader(custom_hf_revision or _PARQUET_REVISION)
+
+    @override
+    def documentation(self) -> str:
+        url = f"https://huggingface.co/datasets/{SOCIAL_I_QA_DATASET_PATH}"
+        return (
+            f"- Link to dataset: [{url}]({url})\n"
+            "- This dataset ships only a loading script on its `main` branch, which newer `datasets` versions no "
+            f"longer support; it is therefore loaded from Hugging Face's auto-generated parquet branch "
+            f"`{_PARQUET_REVISION}` instead."
+        )
+
+
+@final
+class SocialIqaReader(ChoiceReader):
+    """Reads a Social IQa item: the shown question is the context followed by the question, with the correct
+    option at the 1-indexed ``label``."""
+
+    @override
+    def read(self, item: dict[str, Any]) -> ChoiceFields:
+        return ChoiceFields(
+            raw_question=f"{item['context']} {item['question']}".strip(),
+            choices=[item["answerA"], item["answerB"], item["answerC"]],
+            correct_index=int(item["label"]) - 1,
+        )
+
+
+def social_iqa_mc_olmes(dataset: DatasetPolicy | None = None) -> Benchmark:
+    styler = MCStyle(question_prefix="Question: ", cue_text="Answer:", space_prefixed_labels=True)
+    dataset_policy = dataset if dataset is not None else _SocialIqaDataset()
+    return ComposedBenchmark.choice(
+        id="SocialIQAMC_OLMES",
+        reader=SocialIqaReader(),
+        styler=styler,
+        sample_split="train",
+        fewshot_split="train",
+        dataset_policy=dataset_policy,
+        language=Language.ENG,
+    )
+
+
+SOCIAL_IQA_BENCHMARKS: list[Benchmark] = [social_iqa_mc_olmes()]

eval_framework-0.13.0/src/eval_framework/benchmarks/winogrande_ellamind.py → eval_framework-0.13.2/src/eval_framework/benchmarks/winogrande.py

@@ -1,10 +1,9 @@
-"""German Winogrande (EllaMind) tasks.
+"""Winogrande: https://huggingface.co/datasets/allenai/winogrande
 
-https://huggingface.co/datasets/ellamind/winogrande-multilingual
-
-The sentence contains a blank ``_`` filled by ``option1`` or ``option2``; ``answer`` selects the correct
-one. Cloze and MC score the two full "option + suffix" completions; partial evaluation instead scores the
-shared suffix under each option-augmented prefix.
+Pronoun-resolution sentences with a blank ``_`` filled by ``option1`` or ``option2``; ``answer`` selects
+the correct one. The registered task uses partial evaluation: each item becomes two samples that score the
+shared sentence suffix under each option-augmented prefix — ``p(suffix | prefix + option)``. ``WinograndeReader``
+and ``PartialEval`` live here and are reused by the multilingual EllaMind variants.
 """
 
 from typing import TYPE_CHECKING, Any, final, override
@@ -20,7 +19,7 @@ from eval_framework.subjects import ListOfSubjects
 from eval_framework.tasks.base import Language
 from eval_framework.tasks.dataset_loading import DatasetPolicy
 from eval_framework.tasks.dataset_revisions import pinned_by_framework
-from eval_framework.tasks.task_style import ClozeStyle, MCStyle, TaskStyler
+from eval_framework.tasks.task_style import ClozeStyle
 from template_formatting.formatter import Message
 
 if TYPE_CHECKING:
@@ -99,48 +98,21 @@ class PartialEval(EvalKind):
         return completion_text  # partial evaluation is scored by loglikelihood; no completion path
 
 
-def _winogrande_dataset(dataset: DatasetPolicy | None) -> DatasetPolicy:
-    return dataset if dataset is not None else pinned_by_framework("ellamind/winogrande-multilingual")
-
-
-def _winogrande_choice(id: str, styler: TaskStyler, dataset: DatasetPolicy | None = None) -> Benchmark:
-    return ComposedBenchmark.choice(
-        id=id,
-        reader=WinograndeReader(),
-        styler=styler,
-        sample_split="validation",
-        fewshot_split="validation",
-        subjects=ListOfSubjects(["deu"]),
-        dataset_policy=_winogrande_dataset(dataset),
-        language=Language.DEU,
-    )
-
-
-def winogrande_ellamind_cloze_de(dataset: DatasetPolicy | None = None) -> Benchmark:
-    styler = ClozeStyle(question_prefix="", trailing_newline=False, cue_text="")
-    return _winogrande_choice("WINOGRANDE_ELLAMIND_CLOZE_DE", styler, dataset)
-
-
-def winogrande_ellamind_mc_de(dataset: DatasetPolicy | None = None) -> Benchmark:
-    return _winogrande_choice("WINOGRANDE_ELLAMIND_MC_DE", MCStyle.for_language(Language.DEU), dataset)
-
-
-def winogrande_ellamind_partial_eval_de(dataset: DatasetPolicy | None = None) -> Benchmark:
-    # Partial evaluation scores the suffix directly; its few-shot demonstrations render as ordinary cloze.
+def winogrande_cloze(dataset: DatasetPolicy | None = None) -> Benchmark:
+    # "Cloze" is the registered name, but the task is partial evaluation; its few-shot demonstrations
+    # render as ordinary cloze (the prefix, then the correct option + suffix).
     fewshot_styler = ClozeStyle(question_prefix="", trailing_newline=False, cue_text="")
+    dataset_policy = dataset if dataset is not None else pinned_by_framework("allenai/winogrande")
     return ComposedBenchmark.compose(
-        id="WINOGRANDE_ELLAMIND_PARTIAL_EVAL_DE",
+        id="WINOGRANDECloze",
+        display_name="WinograndeCloze",
         kind=PartialEval(),
-        sample_split="validation",
-        fewshot=SampledFewShot(WinograndeReader(), fewshot_styler, "validation"),
-        subjects=ListOfSubjects(["deu"]),
-        dataset_policy=_winogrande_dataset(dataset),
-        language=Language.DEU,
+        sample_split="train",
+        fewshot=SampledFewShot(WinograndeReader(), fewshot_styler, "train"),
+        subjects=ListOfSubjects(["winogrande_xl"]),
+        dataset_policy=dataset_policy,
+        language=Language.ENG,
     )
 
 
-WINOGRANDE_ELLAMIND_BENCHMARKS: list[Benchmark] = [
-    winogrande_ellamind_cloze_de(),
-    winogrande_ellamind_mc_de(),
-    winogrande_ellamind_partial_eval_de(),
-]
+WINOGRANDE_BENCHMARKS: list[Benchmark] = [winogrande_cloze()]

eval_framework-0.13.2/src/eval_framework/benchmarks/winogrande_ellamind.py

@@ -0,0 +1,65 @@
+"""German Winogrande (EllaMind) tasks.
+
+https://huggingface.co/datasets/ellamind/winogrande-multilingual
+
+The sentence contains a blank ``_`` filled by ``option1`` or ``option2``; ``answer`` selects the correct
+one. Cloze and MC score the two full "option + suffix" completions; partial evaluation instead scores the
+shared suffix under each option-augmented prefix.
+"""
+
+from eval_framework.benchmarks.winogrande import PartialEval, WinograndeReader
+from eval_framework.composed import ComposedBenchmark
+from eval_framework.contract import Benchmark
+from eval_framework.fewshot import SampledFewShot
+from eval_framework.subjects import ListOfSubjects
+from eval_framework.tasks.base import Language
+from eval_framework.tasks.dataset_loading import DatasetPolicy
+from eval_framework.tasks.dataset_revisions import pinned_by_framework
+from eval_framework.tasks.task_style import ClozeStyle, MCStyle, TaskStyler
+
+
+def _winogrande_dataset(dataset: DatasetPolicy | None) -> DatasetPolicy:
+    return dataset if dataset is not None else pinned_by_framework("ellamind/winogrande-multilingual")
+
+
+def _winogrande_choice(id: str, styler: TaskStyler, dataset: DatasetPolicy | None = None) -> Benchmark:
+    return ComposedBenchmark.choice(
+        id=id,
+        reader=WinograndeReader(),
+        styler=styler,
+        sample_split="validation",
+        fewshot_split="validation",
+        subjects=ListOfSubjects(["deu"]),
+        dataset_policy=_winogrande_dataset(dataset),
+        language=Language.DEU,
+    )
+
+
+def winogrande_ellamind_cloze_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    styler = ClozeStyle(question_prefix="", trailing_newline=False, cue_text="")
+    return _winogrande_choice("WINOGRANDE_ELLAMIND_CLOZE_DE", styler, dataset)
+
+
+def winogrande_ellamind_mc_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _winogrande_choice("WINOGRANDE_ELLAMIND_MC_DE", MCStyle.for_language(Language.DEU), dataset)
+
+
+def winogrande_ellamind_partial_eval_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    # Partial evaluation scores the suffix directly; its few-shot demonstrations render as ordinary cloze.
+    fewshot_styler = ClozeStyle(question_prefix="", trailing_newline=False, cue_text="")
+    return ComposedBenchmark.compose(
+        id="WINOGRANDE_ELLAMIND_PARTIAL_EVAL_DE",
+        kind=PartialEval(),
+        sample_split="validation",
+        fewshot=SampledFewShot(WinograndeReader(), fewshot_styler, "validation"),
+        subjects=ListOfSubjects(["deu"]),
+        dataset_policy=_winogrande_dataset(dataset),
+        language=Language.DEU,
+    )
+
+
+WINOGRANDE_ELLAMIND_BENCHMARKS: list[Benchmark] = [
+    winogrande_ellamind_cloze_de(),
+    winogrande_ellamind_mc_de(),
+    winogrande_ellamind_partial_eval_de(),
+]

{eval_framework-0.13.0 → eval_framework-0.13.2}/src/eval_framework/composed.py

@@ -5,8 +5,6 @@ import typing
 from collections.abc import Iterable, Sequence
 from typing import TYPE_CHECKING, Any, Self, final, override
 
-from datasets import DatasetDict
-
 from eval_framework.choices import ChoiceReader
 from eval_framework.contract import Benchmark, Eval, ResponseType, Sample
 from eval_framework.eval_kind import Choice, EvalKind, SampleBody
@@ -63,26 +61,19 @@ class ComposedEval(Eval):
         self.language = language
         self.rnd = rnd
 
-    def _shuffle_splits(self, hf_dataset: DatasetDict) -> dict[str, list[dict[str, Any]]]:
-        dataset: dict[str, list[dict[str, Any]]] = {}
-
-        for split, data in hf_dataset.items():
-            if split not in {self.sample_split, self._fewshot.split()}:
-                continue
-
-            data_list = list(data)
+    def _load_dataset(self, load_key: str | None) -> dict[str, list[dict[str, Any]]]:
+        hf_dataset = self.loader.load(load_key)
 
-            if split == self.sample_split:
-                self.rnd.shuffle(data_list)
+        sample_rows = list(hf_dataset[self.sample_split])
+        self.rnd.shuffle(sample_rows)
+        dataset = {self.sample_split: sample_rows}
 
-            dataset[split] = data_list
+        fewshot_split = self._fewshot.split()
+        if self.num_fewshot > 0 and fewshot_split is not None and fewshot_split != self.sample_split:
+            dataset[fewshot_split] = list(hf_dataset[fewshot_split])
 
         return dataset
 
-    def _load_dataset(self, load_key: str | None) -> dict[str, list[dict[str, Any]]]:
-        hf_dataset = self.loader.load(load_key)
-        return self._shuffle_splits(hf_dataset=hf_dataset)
-
     @override
     def iterate_samples(self, num_samples: int | None = None) -> Iterable[Sample]:
         for subject in self._subjects:

{eval_framework-0.13.0 → eval_framework-0.13.2}/src/eval_framework/metrics/completion/math_minerva_completion.py

@@ -2,6 +2,8 @@
 Minerva-style MATH completion metric: exact_match and exact_match_flex.
 """
 
+import logging
+
 from eval_framework.metrics.aggregators.aggregators import PassAtK
 from eval_framework.metrics.base import BaseMetric, MetricResult
 from eval_framework.metrics.completion.minerva_math_utils import (
@@ -11,6 +13,8 @@ from eval_framework.metrics.completion.minerva_math_utils import (
 )
 from eval_framework.shared.types import Completion
 
+logger = logging.getLogger(__name__)
+
 
 class MathMinervaCompletion(BaseMetric[Completion]):
     """
@@ -73,6 +77,8 @@ class MathMinervaCompletion(BaseMetric[Completion]):
             cot_style=self.cot_style,
             relaxed=self.relaxed,
         )
+        # Grading can be killed by the pod's memory limit; the last line seen names the sample.
+        logger.info("Grading %s_%s gold=%r candidates=%r", response.subject, response.id, gold, all_candidates)
 
         exact_match = 0.0
         if all_candidates: