eval-framework 0.11.3__tar.gz → 0.11.8__tar.gz

{eval_framework-0.11.3 → eval_framework-0.11.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.11.3
+Version: 0.11.8
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -215,7 +215,7 @@ Classifier: Topic :: Software Development :: Libraries
 Classifier: Typing :: Typed
 Requires-Dist: pyyaml>=6.0.3,<7
 Requires-Dist: xmltodict>=1.0.4,<1.1
-Requires-Dist: pydantic>=2.13.4,<3
+Requires-Dist: pydantic>=2.13.5,<3
 Requires-Dist: datasets>=5.0.1,<6
 Requires-Dist: pycountry>=26.2.16,<27
 Requires-Dist: nltk>=3.10.3,<4
@@ -229,10 +229,10 @@ Requires-Dist: psycopg2-binary>=2.9.12,<3
 Requires-Dist: sympy>=1.14.0,<2
 Requires-Dist: llm-sandbox[docker]==0.3.44
 Requires-Dist: jsonlines>=4,<5
-Requires-Dist: lxml>=6.1.2,<7
+Requires-Dist: lxml>=6.1.3,<7
 Requires-Dist: python-iso639>=2026.7.23
 Requires-Dist: wandb>=0.29.0,<1
-Requires-Dist: boto3>=1.43.82,<2
+Requires-Dist: boto3>=1.43.87,<2
 Requires-Dist: numpy>=2.5.2
 Requires-Dist: antlr4-python3-runtime==4.11.0
 Requires-Dist: scipy>=1.18.1,<2
@@ -241,13 +241,13 @@ Requires-Dist: eval-framework[determined,api,openai,transformers,accelerate,opti
 Requires-Dist: aleph-alpha-client>=11.5.1 ; extra == 'api'
 Requires-Dist: determined>=0.38.1,<0.39 ; extra == 'determined'
 Requires-Dist: tensorboard==2.21.0 ; extra == 'determined'
-Requires-Dist: openai>=3.5.0,<4 ; extra == 'openai'
+Requires-Dist: openai>=3.7.0,<4 ; extra == 'openai'
 Requires-Dist: tiktoken>=0.14.0,<1 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'optional'
 Requires-Dist: jinja2>=3.1.6,<4 ; extra == 'optional'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'transformers'
-Requires-Dist: torch>=2.13.0,<3 ; extra == 'transformers'
+Requires-Dist: torch>=2.14.0,<3 ; extra == 'transformers'
 Requires-Dist: accelerate>=1.14.0,<2 ; extra == 'transformers'
 Requires-Python: >=3.12, <3.15
 Project-URL: repository, https://github.com/Aleph-Alpha-Research/eval-framework

{eval_framework-0.11.3 → eval_framework-0.11.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.11.3"
+version = "0.11.8"
 description = "Evaluation Framework"
 readme = "README.md"
 requires-python = ">=3.12,<3.15"
@@ -18,7 +18,7 @@ classifiers = [
 dependencies = [
     "pyyaml>=6.0.3,<7",
     "xmltodict>=1.0.4,<1.1",
-    "pydantic>=2.13.4,<3",
+    "pydantic>=2.13.5,<3",
     "datasets>=5.0.1,<6",
     "pycountry>=26.2.16,<27",
     "nltk>=3.10.3,<4",
@@ -32,10 +32,10 @@ dependencies = [
     "sympy>=1.14.0,<2",
     "llm-sandbox[docker]==0.3.44",
     "jsonlines>=4,<5",
-    "lxml>=6.1.2,<7",
+    "lxml>=6.1.3,<7",
     "python-iso639>=2026.7.23",
     "wandb>=0.29.0,<1",
-    "boto3>=1.43.82,<2",
+    "boto3>=1.43.87,<2",
     "numpy>=2.5.2",
     "antlr4-python3-runtime==4.11.0",
     "scipy>=1.18.1,<2",
@@ -54,13 +54,13 @@ determined = [
 ]
 api = ["aleph-alpha-client>=11.5.1"]
 openai = [
-    "openai>=3.5.0,<4",
+    "openai>=3.7.0,<4",
     "tiktoken>=0.14.0,<1",
     "transformers>=4.45.2,<5",
 ]
 transformers = [
     "transformers>=4.45.2,<5",
-    "torch>=2.13.0,<3",
+    "torch>=2.14.0,<3",
     "accelerate>=1.14.0,<2",
 ]
 accelerate = ["accelerate"]
@@ -96,7 +96,7 @@ flash-attn = [
 ]
 
 [build-system]
-requires = ["uv_build>=0.12.7,<0.12.8"]
+requires = ["uv_build>=0.12.9,<0.12.10"]
 build-backend = "uv_build"
 
 [tool.uv]

{eval_framework-0.11.3 → eval_framework-0.11.8}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.11.3"
+version = "0.11.8"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -22,7 +22,7 @@ classifiers = [
 dependencies = [
   "pyyaml>=6.0.3,<7",
   "xmltodict>=1.0.4,<1.1",
-  "pydantic>=2.13.4,<3",
+  "pydantic>=2.13.5,<3",
   "datasets>=5.0.1,<6",
   "pycountry>=26.2.16,<27",
   "nltk>=3.10.3,<4",
@@ -36,10 +36,10 @@ dependencies = [
   "sympy>=1.14.0,<2",
   "llm-sandbox[docker]==0.3.44",
   "jsonlines>=4,<5",
-  "lxml>=6.1.2,<7",
+  "lxml>=6.1.3,<7",
   "python-iso639>=2026.7.23",
   "wandb>=0.29.0,<1",
-  "boto3>=1.43.82,<2",
+  "boto3>=1.43.87,<2",
   "numpy>=2.5.2",
   # is a dependency of sympy, but not explicitly listed in the requirements.txt
   # https://github.com/sympy/sympy/blob/0204fa34e8f6f6f8ccb4de01209be9a2345c9d6e/doc/src/contributing/dependencies.md?plain=1#L125
@@ -55,13 +55,13 @@ determined = [
 ]
 api = ["aleph-alpha-client>=11.5.1"]
 openai = [
-  "openai>=3.5.0,<4",
+  "openai>=3.7.0,<4",
   "tiktoken>=0.14.0,<1",
   "transformers>=4.45.2,<5",
 ]
 transformers = [
   "transformers>=4.45.2,<5",
-  "torch>=2.13.0,<3",
+  "torch>=2.14.0,<3",
   "accelerate>=1.14.0,<2",
 ]
 accelerate = ["accelerate"]
@@ -102,7 +102,7 @@ flash-attn = [
 ]
 
 [build-system]
-requires = ["uv_build>=0.12.7,<0.12.8"]
+requires = ["uv_build>=0.12.9,<0.12.10"]
 build-backend = "uv_build"
 
 [tool.uv.build-backend]

{eval_framework-0.11.3 → eval_framework-0.11.8}/src/eval_framework/benchmarks/arc_de.py

@@ -5,8 +5,9 @@ from typing import Any, final, override
 from eval_framework.choices import ChoiceFields, ChoiceReader
 from eval_framework.composed import ComposedBenchmark
 from eval_framework.contract import Benchmark
-from eval_framework.subjects import NoSubject
+from eval_framework.eval_kind import Choice
 from eval_framework.tasks.base import Language
+from eval_framework.tasks.dataset_loading import DatasetPolicy
 from eval_framework.tasks.dataset_revisions import pinned_by_framework
 from eval_framework.tasks.task_style import ClozeStyle, answer_key_to_index
 
@@ -28,16 +29,22 @@ class ArcDeReader(ChoiceReader):
         )
 
 
-# ARC-DE as cloze/ranked classification.
-# https://huggingface.co/datasets/LeoLM/ArcChallenge_de
-ARC_DE_BENCHMARK: Benchmark = ComposedBenchmark.compose(
-    id="ARC_DE",
-    display_name="ARC German",
-    styler=ClozeStyle(question_prefix="Frage: ", cue_text="Antwort:"),
-    reader=ArcDeReader(),
-    sample_split="test",
-    fewshot_split="validation",
-    subjects=NoSubject(),
-    dataset_policy=pinned_by_framework("LeoLM/ArcChallenge_de"),
-    language=Language.DEU,
-)
+def arc_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    """ARC-DE as cloze/ranked classification.
+
+    https://huggingface.co/datasets/LeoLM/ArcChallenge_de
+    """
+    kind = Choice(reader=ArcDeReader(), styler=ClozeStyle(question_prefix="Frage: ", cue_text="Antwort:"))
+    dataset_policy = dataset if dataset is not None else pinned_by_framework("LeoLM/ArcChallenge_de")
+    return ComposedBenchmark.compose(
+        id="ARC_DE",
+        display_name="ARC German",
+        kind=kind,
+        sample_split="test",
+        fewshot_split="validation",
+        dataset_policy=dataset_policy,
+        language=Language.DEU,
+    )
+
+
+ARC_DE_BENCHMARK: Benchmark = arc_de()

{eval_framework-0.11.3 → eval_framework-0.11.8}/src/eval_framework/benchmarks/csqa_ellamind.py

@@ -10,8 +10,10 @@ from typing import Any, Literal, final, override
 from eval_framework.choices import ChoiceFields, ChoiceReader
 from eval_framework.composed import ComposedBenchmark
 from eval_framework.contract import Benchmark
+from eval_framework.eval_kind import Choice
 from eval_framework.subjects import ListOfSubjects
 from eval_framework.tasks.base import Language
+from eval_framework.tasks.dataset_loading import DatasetPolicy
 from eval_framework.tasks.dataset_revisions import pinned_by_framework
 from eval_framework.tasks.task_style import BPBStyle, ClozeStyle, MCStyle, TaskStyler, shuffle_correct_with_distractors
 
@@ -34,30 +36,50 @@ class CsqaReader(ChoiceReader):
         return ChoiceFields(raw_question=item["question"], choices=choices, correct_index=correct_index)
 
 
-# One styler per format, all sharing the German prefix/cue. The easy/hard distractor axis belongs to
-# the reader, so it is orthogonal to the styler choice.
-CSQA_ELLAMIND_CLOZE_STYLER = ClozeStyle.for_language(Language.DEU)
-CSQA_ELLAMIND_MC_STYLER = MCStyle.for_language(Language.DEU)
-CSQA_ELLAMIND_BPB_STYLER = BPBStyle.for_language(Language.DEU)
-
-
-def _csqa_ellamind_benchmark(id: str, styler: TaskStyler, distractor_level: Literal["easy", "hard"]) -> Benchmark:
+def _csqa_ellamind_benchmark(
+    id: str, styler: TaskStyler, distractor_level: Literal["easy", "hard"], dataset: DatasetPolicy | None = None
+) -> Benchmark:
+    kind = Choice(reader=CsqaReader(distractor_level), styler=styler)
+    dataset_policy = dataset if dataset is not None else pinned_by_framework("ellamind/csqa-multilingual")
     return ComposedBenchmark.compose(
         id=id,
-        styler=styler,
-        reader=CsqaReader(distractor_level),
+        kind=kind,
         sample_split="validation",
         fewshot_split="validation",
         subjects=ListOfSubjects(["deu"]),
-        dataset_policy=pinned_by_framework("ellamind/csqa-multilingual"),
+        dataset_policy=dataset_policy,
         language=Language.DEU,
     )
 
 
+def csqa_ellamind_mc_easy_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _csqa_ellamind_benchmark("CSQA_ELLAMIND_MC_EASY_DE", MCStyle.for_language(Language.DEU), "easy", dataset)
+
+
+def csqa_ellamind_mc_hard_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _csqa_ellamind_benchmark("CSQA_ELLAMIND_MC_HARD_DE", MCStyle.for_language(Language.DEU), "hard", dataset)
+
+
+def csqa_ellamind_cloze_easy_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _csqa_ellamind_benchmark(
+        "CSQA_ELLAMIND_CLOZE_EASY_DE", ClozeStyle.for_language(Language.DEU), "easy", dataset
+    )
+
+
+def csqa_ellamind_cloze_hard_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _csqa_ellamind_benchmark(
+        "CSQA_ELLAMIND_CLOZE_HARD_DE", ClozeStyle.for_language(Language.DEU), "hard", dataset
+    )
+
+
+def csqa_ellamind_bpb_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _csqa_ellamind_benchmark("CSQA_ELLAMIND_BPB_DE", BPBStyle.for_language(Language.DEU), "easy", dataset)
+
+
 CSQA_ELLAMIND_BENCHMARKS: list[Benchmark] = [
-    _csqa_ellamind_benchmark("CSQA_ELLAMIND_MC_EASY_DE", CSQA_ELLAMIND_MC_STYLER, "easy"),
-    _csqa_ellamind_benchmark("CSQA_ELLAMIND_MC_HARD_DE", CSQA_ELLAMIND_MC_STYLER, "hard"),
-    _csqa_ellamind_benchmark("CSQA_ELLAMIND_CLOZE_EASY_DE", CSQA_ELLAMIND_CLOZE_STYLER, "easy"),
-    _csqa_ellamind_benchmark("CSQA_ELLAMIND_CLOZE_HARD_DE", CSQA_ELLAMIND_CLOZE_STYLER, "hard"),
-    _csqa_ellamind_benchmark("CSQA_ELLAMIND_BPB_DE", CSQA_ELLAMIND_BPB_STYLER, "easy"),
+    csqa_ellamind_mc_easy_de(),
+    csqa_ellamind_mc_hard_de(),
+    csqa_ellamind_cloze_easy_de(),
+    csqa_ellamind_cloze_hard_de(),
+    csqa_ellamind_bpb_de(),
 ]

eval_framework-0.11.8/src/eval_framework/benchmarks/gpqa_ellamind.py

@@ -0,0 +1,90 @@
+"""German GPQA (Graduate-level Professional QA, EllaMind) tasks.
+
+https://huggingface.co/datasets/ellamind/gpqa-multilingual
+
+GPQA uses a single distractor set (``incorrect_answers``). The diamond variants restrict evaluation to
+the diamond subset — the 198 hardest questions (``is_diamond``) from the original GPQA-Diamond benchmark.
+"""
+
+from typing import Any, final, override
+
+from eval_framework.choices import ChoiceFields, ChoiceReader
+from eval_framework.composed import ComposedBenchmark
+from eval_framework.contract import Benchmark
+from eval_framework.eval_kind import Choice
+from eval_framework.subjects import ListOfSubjects
+from eval_framework.tasks.base import Language
+from eval_framework.tasks.dataset_loading import DatasetPolicy, Subset
+from eval_framework.tasks.dataset_revisions import pinned_by_framework
+from eval_framework.tasks.task_style import BPBStyle, ClozeStyle, MCStyle, TaskStyler, shuffle_correct_with_distractors
+
+
+@final
+class GpqaReader(ChoiceReader):
+    """Reads a GPQA item: a single ``incorrect_answers`` distractor set, shuffled in with the correct answer."""
+
+    @override
+    def read(self, item: dict[str, Any]) -> ChoiceFields:
+        choices, correct_index = shuffle_correct_with_distractors(
+            correct=item["correct_answer"],
+            distractors=item["incorrect_answers"],
+            seed_text=item["question"] + item["correct_answer"],
+        )
+        return ChoiceFields(raw_question=item["question"], choices=choices, correct_index=correct_index)
+
+
+def _gpqa_ellamind_benchmark(id: str, styler: TaskStyler, dataset: DatasetPolicy | None = None) -> Benchmark:
+    kind = Choice(reader=GpqaReader(), styler=styler)
+    dataset_policy = dataset if dataset is not None else pinned_by_framework("ellamind/gpqa-multilingual")
+    return ComposedBenchmark.compose(
+        id=id,
+        kind=kind,
+        sample_split="train",
+        fewshot_split="train",
+        subjects=ListOfSubjects(["deu"]),
+        dataset_policy=dataset_policy,
+        language=Language.DEU,
+    )
+
+
+def _gpqa_ellamind_diamond_benchmark(id: str, styler: TaskStyler, dataset: DatasetPolicy | None = None) -> Benchmark:
+    source = dataset if dataset is not None else pinned_by_framework("ellamind/gpqa-multilingual")
+    return _gpqa_ellamind_benchmark(id, styler, Subset(source, keep=lambda row: row["is_diamond"]))
+
+
+def gpqa_ellamind_mc_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _gpqa_ellamind_benchmark("GPQA_ELLAMIND_MC_DE", MCStyle.for_language(Language.DEU), dataset)
+
+
+def gpqa_ellamind_cloze_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _gpqa_ellamind_benchmark("GPQA_ELLAMIND_CLOZE_DE", ClozeStyle.for_language(Language.DEU), dataset)
+
+
+def gpqa_ellamind_bpb_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _gpqa_ellamind_benchmark("GPQA_ELLAMIND_BPB_DE", BPBStyle.for_language(Language.DEU), dataset)
+
+
+def gpqa_ellamind_diamond_mc_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _gpqa_ellamind_diamond_benchmark("GPQA_ELLAMIND_DIAMOND_MC_DE", MCStyle.for_language(Language.DEU), dataset)
+
+
+def gpqa_ellamind_diamond_cloze_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _gpqa_ellamind_diamond_benchmark(
+        "GPQA_ELLAMIND_DIAMOND_CLOZE_DE", ClozeStyle.for_language(Language.DEU), dataset
+    )
+
+
+def gpqa_ellamind_diamond_bpb_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _gpqa_ellamind_diamond_benchmark(
+        "GPQA_ELLAMIND_DIAMOND_BPB_DE", BPBStyle.for_language(Language.DEU), dataset
+    )
+
+
+GPQA_ELLAMIND_BENCHMARKS: list[Benchmark] = [
+    gpqa_ellamind_mc_de(),
+    gpqa_ellamind_cloze_de(),
+    gpqa_ellamind_diamond_mc_de(),
+    gpqa_ellamind_diamond_cloze_de(),
+    gpqa_ellamind_bpb_de(),
+    gpqa_ellamind_diamond_bpb_de(),
+]

eval_framework-0.11.8/src/eval_framework/benchmarks/hellaswag_ellamind.py

@@ -0,0 +1,84 @@
+"""German HellaSwag (EllaMind) tasks.
+
+https://huggingface.co/datasets/ellamind/hellaswag-multilingual
+
+HellaSwag is a sentence-completion task: the prompt is a partial sentence (``"{activity}: {context}"``)
+and the model scores full-sentence endings. There is no natural MC variant. HellaSwag supplies separate
+easy and hard distractors.
+"""
+
+from typing import Any, Literal, final, override
+
+from eval_framework.choices import ChoiceFields, ChoiceReader
+from eval_framework.composed import ComposedBenchmark
+from eval_framework.contract import Benchmark
+from eval_framework.eval_kind import Choice
+from eval_framework.subjects import ListOfSubjects
+from eval_framework.tasks.base import Language
+from eval_framework.tasks.dataset_loading import DatasetPolicy
+from eval_framework.tasks.dataset_revisions import pinned_by_framework
+from eval_framework.tasks.task_style import BPBStyle, ClozeStyle, TaskStyler, shuffle_correct_with_distractors
+
+
+@final
+class HellaswagReader(ChoiceReader):
+    """Reads a HellaSwag item: the partial sentence ``"{activity}: {context}"``, with the easy/hard
+    full-sentence endings for the level shuffled in with the correct ending."""
+
+    def __init__(self, distractor_level: Literal["easy", "hard"]) -> None:
+        self._distractor_level = distractor_level
+
+    @override
+    def read(self, item: dict[str, Any]) -> ChoiceFields:
+        distractors = item["easy_distractors"] if self._distractor_level == "easy" else item["hard_distractors"]
+        choices, correct_index = shuffle_correct_with_distractors(
+            correct=item["correct_ending"],
+            distractors=distractors,
+            seed_text=item["context"] + item["correct_ending"],
+        )
+        return ChoiceFields(
+            raw_question=f"{item['activity'].strip()}: {item['context'].strip()}",
+            choices=choices,
+            correct_index=correct_index,
+        )
+
+
+def _hellaswag_ellamind_benchmark(
+    id: str, styler: TaskStyler, distractor_level: Literal["easy", "hard"], dataset: DatasetPolicy | None = None
+) -> Benchmark:
+    kind = Choice(reader=HellaswagReader(distractor_level), styler=styler)
+    dataset_policy = dataset if dataset is not None else pinned_by_framework("ellamind/hellaswag-multilingual")
+    return ComposedBenchmark.compose(
+        id=id,
+        kind=kind,
+        sample_split="validation",
+        fewshot_split="validation",
+        subjects=ListOfSubjects(["deu"]),
+        dataset_policy=dataset_policy,
+        language=Language.DEU,
+    )
+
+
+# Sentence-completion: no question prefix, no cue, the continuation follows the context directly.
+def _cloze_completion_style() -> ClozeStyle:
+    return ClozeStyle(question_prefix="", trailing_newline=False, cue_text="")
+
+
+def hellaswag_ellamind_easy_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _hellaswag_ellamind_benchmark("HELLASWAG_ELLAMIND_EASY_DE", _cloze_completion_style(), "easy", dataset)
+
+
+def hellaswag_ellamind_hard_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _hellaswag_ellamind_benchmark("HELLASWAG_ELLAMIND_HARD_DE", _cloze_completion_style(), "hard", dataset)
+
+
+def hellaswag_ellamind_bpb_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    styler = BPBStyle(question_prefix="", trailing_newline=False, cue_text="")
+    return _hellaswag_ellamind_benchmark("HELLASWAG_ELLAMIND_BPB_DE", styler, "easy", dataset)
+
+
+HELLASWAG_ELLAMIND_BENCHMARKS: list[Benchmark] = [
+    hellaswag_ellamind_easy_de(),
+    hellaswag_ellamind_hard_de(),
+    hellaswag_ellamind_bpb_de(),
+]

eval_framework-0.11.8/src/eval_framework/benchmarks/hle_ellamind.py

@@ -0,0 +1,83 @@
+"""German HLE (Humanity's Last Exam, EllaMind) tasks.
+
+https://huggingface.co/datasets/ellamind/hle-multilingual
+
+HLE uses a single distractor set (``incorrect_answers``). The NATIVE variants restrict evaluation to the
+items that are natively multiple-choice in the original benchmark (``answer_type == "multipleChoice"``).
+"""
+
+from typing import Any, final, override
+
+from eval_framework.choices import ChoiceFields, ChoiceReader
+from eval_framework.composed import ComposedBenchmark
+from eval_framework.contract import Benchmark
+from eval_framework.eval_kind import Choice
+from eval_framework.subjects import ListOfSubjects
+from eval_framework.tasks.base import Language
+from eval_framework.tasks.dataset_loading import DatasetPolicy, Subset
+from eval_framework.tasks.dataset_revisions import pinned_by_framework
+from eval_framework.tasks.task_style import BPBStyle, ClozeStyle, MCStyle, TaskStyler, shuffle_correct_with_distractors
+
+
+@final
+class HleReader(ChoiceReader):
+    """Reads an HLE item: a single ``incorrect_answers`` distractor set, shuffled in with the correct answer."""
+
+    @override
+    def read(self, item: dict[str, Any]) -> ChoiceFields:
+        choices, correct_index = shuffle_correct_with_distractors(
+            correct=item["correct_answer"],
+            distractors=item["incorrect_answers"],
+            seed_text=item["question"] + item["correct_answer"],
+        )
+        return ChoiceFields(raw_question=item["question"], choices=choices, correct_index=correct_index)
+
+
+def _hle_ellamind_benchmark(id: str, styler: TaskStyler, dataset: DatasetPolicy | None = None) -> Benchmark:
+    kind = Choice(reader=HleReader(), styler=styler)
+    dataset_policy = dataset if dataset is not None else pinned_by_framework("ellamind/hle-multilingual")
+    return ComposedBenchmark.compose(
+        id=id,
+        kind=kind,
+        sample_split="test",
+        fewshot_split="test",
+        subjects=ListOfSubjects(["deu"]),
+        dataset_policy=dataset_policy,
+        language=Language.DEU,
+    )
+
+
+def _hle_ellamind_native_benchmark(id: str, styler: TaskStyler, dataset: DatasetPolicy | None = None) -> Benchmark:
+    source = dataset if dataset is not None else pinned_by_framework("ellamind/hle-multilingual")
+    return _hle_ellamind_benchmark(id, styler, Subset(source, keep=lambda row: row["answer_type"] == "multipleChoice"))
+
+
+def hle_ellamind_mc_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _hle_ellamind_benchmark("HLE_ELLAMIND_MC_DE", MCStyle.for_language(Language.DEU), dataset)
+
+
+def hle_ellamind_cloze_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _hle_ellamind_benchmark("HLE_ELLAMIND_CLOZE_DE", ClozeStyle.for_language(Language.DEU), dataset)
+
+
+def hle_ellamind_mc_native_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _hle_ellamind_native_benchmark("HLE_ELLAMIND_MC_NATIVE_DE", MCStyle.for_language(Language.DEU), dataset)
+
+
+def hle_ellamind_cloze_native_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _hle_ellamind_native_benchmark(
+        "HLE_ELLAMIND_CLOZE_NATIVE_DE", ClozeStyle.for_language(Language.DEU), dataset
+    )
+
+
+def hle_ellamind_bpb_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _hle_ellamind_benchmark("HLE_ELLAMIND_BPB_DE", BPBStyle.for_language(Language.DEU), dataset)
+
+
+HLE_ELLAMIND_BENCHMARKS: list[Benchmark] = [
+    hle_ellamind_mc_de(),
+    hle_ellamind_cloze_de(),
+    hle_ellamind_mc_native_de(),
+    hle_ellamind_cloze_native_de(),
+    hle_ellamind_bpb_de(),
+]

eval_framework-0.11.8/src/eval_framework/benchmarks/piqa_ellamind.py

@@ -0,0 +1,95 @@
+"""German PIQA (EllaMind) tasks.
+
+https://huggingface.co/datasets/ellamind/piqa-multilingual
+
+PIQA supplies separate easy and hard distractors.
+"""
+
+from typing import Any, Literal, final, override
+
+from eval_framework.choices import ChoiceFields, ChoiceReader
+from eval_framework.composed import ComposedBenchmark
+from eval_framework.contract import Benchmark
+from eval_framework.eval_kind import Choice
+from eval_framework.subjects import ListOfSubjects
+from eval_framework.tasks.base import Language
+from eval_framework.tasks.dataset_loading import DatasetPolicy
+from eval_framework.tasks.dataset_revisions import pinned_by_framework
+from eval_framework.tasks.task_style import BPBStyle, ClozeStyle, MCStyle, TaskStyler, shuffle_correct_with_distractors
+
+
+@final
+class PiqaReader(ChoiceReader):
+    """Reads a PIQA item: a single easy/hard distractor per level, shuffled in with the correct solution."""
+
+    def __init__(self, distractor_level: Literal["easy", "hard"]) -> None:
+        self._distractor_level = distractor_level
+
+    @override
+    def read(self, item: dict[str, Any]) -> ChoiceFields:
+        distractor = item["easy_distractor"] if self._distractor_level == "easy" else item["hard_distractor"]
+        choices, correct_index = shuffle_correct_with_distractors(
+            correct=item["correct_solution"],
+            distractors=[distractor],
+            seed_text=item["goal"] + item["correct_solution"],
+        )
+        return ChoiceFields(raw_question=item["goal"], choices=choices, correct_index=correct_index)
+
+
+_QUESTION_PREFIX = "Ziel: "
+_CUE_TEXT = "Antwort:"
+
+
+def _piqa_ellamind_benchmark(
+    id: str, styler: TaskStyler, distractor_level: Literal["easy", "hard"], dataset: DatasetPolicy | None = None
+) -> Benchmark:
+    kind = Choice(reader=PiqaReader(distractor_level), styler=styler)
+    dataset_policy = dataset if dataset is not None else pinned_by_framework("ellamind/piqa-multilingual")
+    return ComposedBenchmark.compose(
+        id=id,
+        kind=kind,
+        sample_split="validation",
+        fewshot_split="validation",
+        subjects=ListOfSubjects(["deu"]),
+        dataset_policy=dataset_policy,
+        language=Language.DEU,
+    )
+
+
+def piqa_ellamind_cloze_easy_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _piqa_ellamind_benchmark(
+        "PIQA_ELLAMIND_CLOZE_EASY_DE", ClozeStyle(question_prefix=_QUESTION_PREFIX, cue_text=_CUE_TEXT), "easy", dataset
+    )
+
+
+def piqa_ellamind_cloze_hard_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _piqa_ellamind_benchmark(
+        "PIQA_ELLAMIND_CLOZE_HARD_DE", ClozeStyle(question_prefix=_QUESTION_PREFIX, cue_text=_CUE_TEXT), "hard", dataset
+    )
+
+
+def piqa_ellamind_mc_easy_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _piqa_ellamind_benchmark(
+        "PIQA_ELLAMIND_MC_EASY_DE", MCStyle(question_prefix=_QUESTION_PREFIX, cue_text=_CUE_TEXT), "easy", dataset
+    )
+
+
+def piqa_ellamind_mc_hard_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _piqa_ellamind_benchmark(
+        "PIQA_ELLAMIND_MC_HARD_DE", MCStyle(question_prefix=_QUESTION_PREFIX, cue_text=_CUE_TEXT), "hard", dataset
+    )
+
+
+def piqa_ellamind_bpb_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _piqa_ellamind_benchmark(
+        "PIQA_ELLAMIND_BPB_DE", BPBStyle(question_prefix=_QUESTION_PREFIX, cue_text=_CUE_TEXT), "easy", dataset
+    )
+
+
+PIQA_ELLAMIND_BENCHMARKS: list[Benchmark] = [
+    piqa_ellamind_cloze_easy_de(),
+    piqa_ellamind_cloze_hard_de(),
+    piqa_ellamind_mc_easy_de(),
+    piqa_ellamind_mc_hard_de(),
+    piqa_ellamind_bpb_de(),
+]