eval-framework 0.12.0__tar.gz → 0.13.0__tar.gz

{eval_framework-0.12.0 → eval_framework-0.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.12.0
+Version: 0.13.0
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License

{eval_framework-0.12.0 → eval_framework-0.13.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.12.0"
+version = "0.13.0"
 description = "Evaluation Framework"
 readme = "README.md"
 requires-python = ">=3.12,<3.15"

{eval_framework-0.12.0 → eval_framework-0.13.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.12.0"
+version = "0.13.0"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }

{eval_framework-0.12.0 → eval_framework-0.13.0}/src/eval_framework/benchmarks/arc_de.py

@@ -5,7 +5,6 @@ from typing import Any, final, override
 from eval_framework.choices import ChoiceFields, ChoiceReader
 from eval_framework.composed import ComposedBenchmark
 from eval_framework.contract import Benchmark
-from eval_framework.eval_kind import Choice
 from eval_framework.tasks.base import Language
 from eval_framework.tasks.dataset_loading import DatasetPolicy
 from eval_framework.tasks.dataset_revisions import pinned_by_framework
@@ -34,12 +33,12 @@ def arc_de(dataset: DatasetPolicy | None = None) -> Benchmark:
 
     https://huggingface.co/datasets/LeoLM/ArcChallenge_de
     """
-    kind = Choice(reader=ArcDeReader(), styler=ClozeStyle(question_prefix="Frage: ", cue_text="Antwort:"))
     dataset_policy = dataset if dataset is not None else pinned_by_framework("LeoLM/ArcChallenge_de")
-    return ComposedBenchmark.compose(
+    return ComposedBenchmark.choice(
         id="ARC_DE",
         display_name="ARC German",
-        kind=kind,
+        reader=ArcDeReader(),
+        styler=ClozeStyle(question_prefix="Frage: ", cue_text="Antwort:"),
         sample_split="test",
         fewshot_split="validation",
         dataset_policy=dataset_policy,

eval_framework-0.13.0/src/eval_framework/benchmarks/copa.py

@@ -0,0 +1,59 @@
+"""COPA (Choice of Plausible Alternatives): https://huggingface.co/datasets/aps/super_glue
+
+Causal-reasoning items: a premise, a cause/effect cue, and two alternatives. The registered variant is
+OLMES-style: the premise is recast as a sentence stem (its final period replaced by the causal connector),
+the two alternatives are shown as space-prefixed lettered options (" A. …"), and the model is scored over
+the letter labels. The prompt carries no assistant cue — the options directly continue the stem.
+"""
+
+from typing import Any, final, override
+
+from eval_framework.choices import ChoiceFields, ChoiceReader
+from eval_framework.composed import ComposedBenchmark
+from eval_framework.contract import Benchmark
+from eval_framework.subjects import ListOfSubjects
+from eval_framework.tasks.base import Language
+from eval_framework.tasks.dataset_loading import DatasetPolicy
+from eval_framework.tasks.dataset_revisions import pinned_by_framework
+from eval_framework.tasks.task_style import MCStyle
+
+# The premise becomes a sentence stem: its trailing period is replaced by the causal connector, so each
+# alternative continues it (e.g. "The man broke his toe because" + " he dropped a hammer on it.").
+_COPA_CONNECTOR = {"cause": "because", "effect": "therefore"}
+
+
+def _decapitalize(text: str) -> str:
+    return text[0].lower() + text[1:]
+
+
+@final
+class CopaReader(ChoiceReader):
+    """Reads a COPA item: the premise recast as a stem (final period → connector) and its two alternatives,
+    each lower-cased to continue the stem."""
+
+    @override
+    def read(self, item: dict[str, Any]) -> ChoiceFields:
+        premise = item["premise"].strip()[:-1] + f" {_COPA_CONNECTOR[item['question']]}"
+        return ChoiceFields(
+            raw_question=premise,
+            choices=[_decapitalize(item["choice1"]), _decapitalize(item["choice2"])],
+            correct_index=item["label"],
+        )
+
+
+def copa_mc_olmes(dataset: DatasetPolicy | None = None) -> Benchmark:
+    styler = MCStyle(question_prefix="", cue_text="", space_prefixed_labels=True)
+    dataset_policy = dataset if dataset is not None else pinned_by_framework("aps/super_glue")
+    return ComposedBenchmark.choice(
+        id="COPA_OLMES",
+        reader=CopaReader(),
+        styler=styler,
+        sample_split="validation",
+        fewshot_split="test",
+        subjects=ListOfSubjects(["copa"]),
+        dataset_policy=dataset_policy,
+        language=Language.ENG,
+    )
+
+
+COPA_BENCHMARKS: list[Benchmark] = [copa_mc_olmes()]

eval_framework-0.13.0/src/eval_framework/benchmarks/csqa.py

@@ -0,0 +1,46 @@
+"""CommonsenseQA (English): https://huggingface.co/datasets/tau/commonsense_qa
+
+Five-way multiple-choice commonsense questions. The registered variant is OLMES-style: the options are
+shown as space-prefixed lettered choices (" A. …") and the model is scored over the letter labels.
+"""
+
+from typing import Any, final, override
+
+from eval_framework.choices import ChoiceFields, ChoiceReader
+from eval_framework.composed import ComposedBenchmark
+from eval_framework.contract import Benchmark
+from eval_framework.tasks.base import Language
+from eval_framework.tasks.dataset_loading import DatasetPolicy
+from eval_framework.tasks.dataset_revisions import pinned_by_framework
+from eval_framework.tasks.task_style import MCStyle
+
+
+@final
+class CommonsenseQaReader(ChoiceReader):
+    """Reads a CommonsenseQA item: the question and its labelled options, with the correct one at ``answerKey``."""
+
+    @override
+    def read(self, item: dict[str, Any]) -> ChoiceFields:
+        labels = item["choices"]["label"]
+        return ChoiceFields(
+            raw_question=item["question"],
+            choices=item["choices"]["text"],
+            correct_index=labels.index(item["answerKey"]),
+        )
+
+
+def csqa_mc_olmes(dataset: DatasetPolicy | None = None) -> Benchmark:
+    styler = MCStyle(question_prefix="Question: ", cue_text="Answer:", space_prefixed_labels=True)
+    dataset_policy = dataset if dataset is not None else pinned_by_framework("tau/commonsense_qa")
+    return ComposedBenchmark.choice(
+        id="CommonsenseQAMC_OLMES",
+        reader=CommonsenseQaReader(),
+        styler=styler,
+        sample_split="train",
+        fewshot_split="train",
+        dataset_policy=dataset_policy,
+        language=Language.ENG,
+    )
+
+
+CSQA_BENCHMARKS: list[Benchmark] = [csqa_mc_olmes()]

{eval_framework-0.12.0 → eval_framework-0.13.0}/src/eval_framework/benchmarks/csqa_ellamind.py

@@ -10,7 +10,6 @@ from typing import Any, Literal, final, override
 from eval_framework.choices import ChoiceFields, ChoiceReader
 from eval_framework.composed import ComposedBenchmark
 from eval_framework.contract import Benchmark
-from eval_framework.eval_kind import Choice
 from eval_framework.subjects import ListOfSubjects
 from eval_framework.tasks.base import Language
 from eval_framework.tasks.dataset_loading import DatasetPolicy
@@ -39,11 +38,11 @@ class CsqaReader(ChoiceReader):
 def _csqa_ellamind_benchmark(
     id: str, styler: TaskStyler, distractor_level: Literal["easy", "hard"], dataset: DatasetPolicy | None = None
 ) -> Benchmark:
-    kind = Choice(reader=CsqaReader(distractor_level), styler=styler)
     dataset_policy = dataset if dataset is not None else pinned_by_framework("ellamind/csqa-multilingual")
-    return ComposedBenchmark.compose(
+    return ComposedBenchmark.choice(
         id=id,
-        kind=kind,
+        reader=CsqaReader(distractor_level),
+        styler=styler,
         sample_split="validation",
         fewshot_split="validation",
         subjects=ListOfSubjects(["deu"]),

{eval_framework-0.12.0 → eval_framework-0.13.0}/src/eval_framework/benchmarks/goldenswag.py

@@ -7,7 +7,6 @@ A curated subset of HellaSwag, read identically (sentence completion, scored as
 from eval_framework.benchmarks.hellaswag import HellaswagReader
 from eval_framework.composed import ComposedBenchmark
 from eval_framework.contract import Benchmark
-from eval_framework.eval_kind import Choice
 from eval_framework.tasks.base import Language
 from eval_framework.tasks.dataset_loading import DatasetPolicy
 from eval_framework.tasks.dataset_revisions import pinned_by_framework
@@ -21,11 +20,11 @@ _IDK_PREAMBLE = (
 
 
 def _goldenswag_benchmark(id: str, styler: TaskStyler, dataset: DatasetPolicy | None = None) -> Benchmark:
-    kind = Choice(reader=HellaswagReader(), styler=styler)
     dataset_policy = dataset if dataset is not None else pinned_by_framework("PleIAs/GoldenSwag")
-    return ComposedBenchmark.compose(
+    return ComposedBenchmark.choice(
         id=id,
-        kind=kind,
+        reader=HellaswagReader(),
+        styler=styler,
         sample_split="validation",
         fewshot_split="validation",
         dataset_policy=dataset_policy,

{eval_framework-0.12.0 → eval_framework-0.13.0}/src/eval_framework/benchmarks/gpqa_ellamind.py

@@ -11,7 +11,6 @@ from typing import Any, final, override
 from eval_framework.choices import ChoiceFields, ChoiceReader
 from eval_framework.composed import ComposedBenchmark
 from eval_framework.contract import Benchmark
-from eval_framework.eval_kind import Choice
 from eval_framework.subjects import ListOfSubjects
 from eval_framework.tasks.base import Language
 from eval_framework.tasks.dataset_loading import DatasetPolicy, Subset
@@ -34,11 +33,11 @@ class GpqaReader(ChoiceReader):
 
 
 def _gpqa_ellamind_benchmark(id: str, styler: TaskStyler, dataset: DatasetPolicy | None = None) -> Benchmark:
-    kind = Choice(reader=GpqaReader(), styler=styler)
     dataset_policy = dataset if dataset is not None else pinned_by_framework("ellamind/gpqa-multilingual")
-    return ComposedBenchmark.compose(
+    return ComposedBenchmark.choice(
         id=id,
-        kind=kind,
+        reader=GpqaReader(),
+        styler=styler,
         sample_split="train",
         fewshot_split="train",
         subjects=ListOfSubjects(["deu"]),

{eval_framework-0.12.0 → eval_framework-0.13.0}/src/eval_framework/benchmarks/hellaswag.py

@@ -12,7 +12,6 @@ from typing import Any, final, override
 from eval_framework.choices import ChoiceFields, ChoiceReader
 from eval_framework.composed import ComposedBenchmark
 from eval_framework.contract import Benchmark
-from eval_framework.eval_kind import Choice
 from eval_framework.tasks.base import Language
 from eval_framework.tasks.dataset_loading import DatasetPolicy
 from eval_framework.tasks.dataset_revisions import pinned_by_framework
@@ -47,11 +46,11 @@ class HellaswagReader(ChoiceReader):
 def _hellaswag_benchmark(
     id: str, *, sample_split: str, fewshot_split: str, dataset: DatasetPolicy | None = None
 ) -> Benchmark:
-    kind = Choice(reader=HellaswagReader(), styler=ClozeStyle(question_prefix="", cue_text="", trailing_newline=False))
     dataset_policy = dataset if dataset is not None else pinned_by_framework("Rowan/hellaswag")
-    return ComposedBenchmark.compose(
+    return ComposedBenchmark.choice(
         id=id,
-        kind=kind,
+        reader=HellaswagReader(),
+        styler=ClozeStyle(question_prefix="", cue_text="", trailing_newline=False),
         sample_split=sample_split,
         fewshot_split=fewshot_split,
         dataset_policy=dataset_policy,

{eval_framework-0.12.0 → eval_framework-0.13.0}/src/eval_framework/benchmarks/hellaswag_ellamind.py

@@ -12,7 +12,6 @@ from typing import Any, Literal, final, override
 from eval_framework.choices import ChoiceFields, ChoiceReader
 from eval_framework.composed import ComposedBenchmark
 from eval_framework.contract import Benchmark
-from eval_framework.eval_kind import Choice
 from eval_framework.subjects import ListOfSubjects
 from eval_framework.tasks.base import Language
 from eval_framework.tasks.dataset_loading import DatasetPolicy
@@ -46,11 +45,11 @@ class HellaswagReader(ChoiceReader):
 def _hellaswag_ellamind_benchmark(
     id: str, styler: TaskStyler, distractor_level: Literal["easy", "hard"], dataset: DatasetPolicy | None = None
 ) -> Benchmark:
-    kind = Choice(reader=HellaswagReader(distractor_level), styler=styler)
     dataset_policy = dataset if dataset is not None else pinned_by_framework("ellamind/hellaswag-multilingual")
-    return ComposedBenchmark.compose(
+    return ComposedBenchmark.choice(
         id=id,
-        kind=kind,
+        reader=HellaswagReader(distractor_level),
+        styler=styler,
         sample_split="validation",
         fewshot_split="validation",
         subjects=ListOfSubjects(["deu"]),

{eval_framework-0.12.0 → eval_framework-0.13.0}/src/eval_framework/benchmarks/hle_ellamind.py

@@ -11,7 +11,6 @@ from typing import Any, final, override
 from eval_framework.choices import ChoiceFields, ChoiceReader
 from eval_framework.composed import ComposedBenchmark
 from eval_framework.contract import Benchmark
-from eval_framework.eval_kind import Choice
 from eval_framework.subjects import ListOfSubjects
 from eval_framework.tasks.base import Language
 from eval_framework.tasks.dataset_loading import DatasetPolicy, Subset
@@ -34,11 +33,11 @@ class HleReader(ChoiceReader):
 
 
 def _hle_ellamind_benchmark(id: str, styler: TaskStyler, dataset: DatasetPolicy | None = None) -> Benchmark:
-    kind = Choice(reader=HleReader(), styler=styler)
     dataset_policy = dataset if dataset is not None else pinned_by_framework("ellamind/hle-multilingual")
-    return ComposedBenchmark.compose(
+    return ComposedBenchmark.choice(
         id=id,
-        kind=kind,
+        reader=HleReader(),
+        styler=styler,
         sample_split="test",
         fewshot_split="test",
         subjects=ListOfSubjects(["deu"]),

eval_framework-0.13.0/src/eval_framework/benchmarks/medqa.py

@@ -0,0 +1,45 @@
+"""MedQA (English): https://huggingface.co/datasets/davidheineman/medqa-en
+
+Multiple-choice questions from medical licensing exams. The registered variant is OLMES-style: the options
+are shown as space-prefixed lettered choices (" A. …") and the model is scored over the letter labels.
+"""
+
+from typing import Any, final, override
+
+from eval_framework.choices import ChoiceFields, ChoiceReader
+from eval_framework.composed import ComposedBenchmark
+from eval_framework.contract import Benchmark
+from eval_framework.tasks.base import Language
+from eval_framework.tasks.dataset_loading import DatasetPolicy
+from eval_framework.tasks.dataset_revisions import pinned_by_framework
+from eval_framework.tasks.task_style import MCStyle
+
+
+@final
+class MedqaReader(ChoiceReader):
+    """Reads a MedQA item: the exam question and its options, with the correct one at ``answer_idx``."""
+
+    @override
+    def read(self, item: dict[str, Any]) -> ChoiceFields:
+        return ChoiceFields(
+            raw_question=item["question"],
+            choices=item["choices"],
+            correct_index=int(item["answer_idx"]),
+        )
+
+
+def medqa_mc_olmes(dataset: DatasetPolicy | None = None) -> Benchmark:
+    styler = MCStyle(question_prefix="Question: ", cue_text="Answer:", space_prefixed_labels=True)
+    dataset_policy = dataset if dataset is not None else pinned_by_framework("davidheineman/medqa-en")
+    return ComposedBenchmark.choice(
+        id="MedQAMC_OLMES",
+        reader=MedqaReader(),
+        styler=styler,
+        sample_split="test",
+        fewshot_split="train",
+        dataset_policy=dataset_policy,
+        language=Language.ENG,
+    )
+
+
+MEDQA_BENCHMARKS: list[Benchmark] = [medqa_mc_olmes()]

eval_framework-0.13.0/src/eval_framework/benchmarks/mmlu.py

@@ -0,0 +1,266 @@
+"""MMLU: https://huggingface.co/datasets/cais/mmlu
+
+Multiple-choice knowledge questions across 57 subjects (each an HF config, loaded per subject); every
+prompt is prefaced by a subject-templated preamble. The composed variants:
+
+* ``MMLU`` / ``MMLU_OLMES`` — score the letter labels; OLMES adds a space before each option label.
+* ``Full Text MMLU`` — shows the options as a bulleted list and scores the full answer text.
+* ``MMLU_IDK`` — lets the model abstain with ``"?"`` and reports confidence-aware metrics.
+* ``MMLU_COT`` — the model reasons freely and concludes with the answer, which is regex-extracted from
+  the generation (free-form completion, 0-shot).
+"""
+
+import re
+from typing import TYPE_CHECKING, Any, final, override
+
+from eval_framework.choices import ChoiceFields, ChoiceReader
+from eval_framework.composed import ComposedBenchmark
+from eval_framework.contract import Benchmark, ResponseType
+from eval_framework.eval_kind import EvalKind, SampleBody
+from eval_framework.fewshot import NoFewShot
+from eval_framework.metrics.completion.accuracy_completion import AccuracyCompletion
+from eval_framework.shared.types import BaseMetricContext
+from eval_framework.subjects import ListOfSubjects
+from eval_framework.tasks.base import Language
+from eval_framework.tasks.dataset_loading import DatasetPolicy
+from eval_framework.tasks.dataset_revisions import pinned_by_framework
+from eval_framework.tasks.task_style import ClozeStyle, MCStyle, TaskStyler
+from eval_framework.tasks.utils import get_n_letters
+from template_formatting.formatter import Message
+
+if TYPE_CHECKING:
+    from eval_framework.metrics.base import BaseMetric
+
+
+# The 57 MMLU subjects; each is an HF config of ``cais/mmlu`` (also reused by GlobalMMLU).
+MMLU_SUBJECTS = [
+    "abstract_algebra",
+    "anatomy",
+    "astronomy",
+    "business_ethics",
+    "clinical_knowledge",
+    "college_biology",
+    "college_chemistry",
+    "college_computer_science",
+    "college_mathematics",
+    "college_medicine",
+    "college_physics",
+    "computer_security",
+    "conceptual_physics",
+    "econometrics",
+    "electrical_engineering",
+    "elementary_mathematics",
+    "formal_logic",
+    "global_facts",
+    "high_school_biology",
+    "high_school_chemistry",
+    "high_school_computer_science",
+    "high_school_european_history",
+    "high_school_geography",
+    "high_school_government_and_politics",
+    "high_school_macroeconomics",
+    "high_school_mathematics",
+    "high_school_microeconomics",
+    "high_school_physics",
+    "high_school_psychology",
+    "high_school_statistics",
+    "high_school_us_history",
+    "high_school_world_history",
+    "human_aging",
+    "human_sexuality",
+    "international_law",
+    "jurisprudence",
+    "logical_fallacies",
+    "machine_learning",
+    "management",
+    "marketing",
+    "medical_genetics",
+    "miscellaneous",
+    "moral_disputes",
+    "moral_scenarios",
+    "nutrition",
+    "philosophy",
+    "prehistory",
+    "professional_accounting",
+    "professional_law",
+    "professional_medicine",
+    "professional_psychology",
+    "public_relations",
+    "security_studies",
+    "sociology",
+    "us_foreign_policy",
+    "virology",
+    "world_religions",
+]
+
+
+@final
+class MmluReader(ChoiceReader):
+    """Reads an MMLU item: the question and its four answer choices, with the correct one at ``answer``."""
+
+    @override
+    def read(self, item: dict[str, Any]) -> ChoiceFields:
+        return ChoiceFields(
+            raw_question=item["question"].strip(),
+            choices=item["choices"],
+            correct_index=item["answer"],
+        )
+
+
+@final
+class _FullTextMmluStyle(ClozeStyle):
+    """Cloze scoring (full answer text) that additionally shows the options as a bulleted list."""
+
+    @override
+    def get_instruction_text(self, raw_question: str, choices: list[str]) -> str:
+        options = "".join(f"- {choice}\n" for choice in choices)
+        return f"{self.question_prefix}{raw_question}\nPossible answers:\n{options}"
+
+
+def _humanized(subject_label: str) -> str:
+    # Subject labels are underscored config names ("abstract_algebra"); read them as prose.
+    return " ".join(subject_label.split("_"))
+
+
+def _mc_preamble(subject_label: str) -> str:
+    return f"The following are multiple choice questions (with answers) about {_humanized(subject_label)}."
+
+
+def _full_text_preamble(subject_label: str) -> str:
+    return (
+        f"The following are multiple choice questions (with possible answers) about {_humanized(subject_label)}.\n"
+        "Answer with the full text of the correct answer."
+    )
+
+
+def _idk_preamble(subject_label: str) -> str:
+    # Unlike the other variants, the IDK preamble names the subject by its raw (underscored) key.
+    return (
+        f"The following are multiple choice questions (with answers) about {subject_label}. "
+        "Answer only if you are confident, since mistakes may be penalised, while correct answers receive points. "
+        "It is acceptable to answer with '?' if you are unsure, and you will receive 0 points."
+    )
+
+
+@final
+class _MmluCotKind(EvalKind):
+    """MMLU chain-of-thought: the model reasons freely and concludes with "Therefore, the answer is: X",
+    and the answer letter is extracted from the generation. Free-form (completion), 0-shot only."""
+
+    def __init__(self) -> None:
+        self._reader = MmluReader()
+        self._answer_re = re.compile(r"Therefore, the answer is: ([ABCD])")
+
+    @override
+    def response_type(self) -> ResponseType:
+        return ResponseType.COMPLETION
+
+    @override
+    def metrics(self) -> list[type["BaseMetric"]]:
+        return [AccuracyCompletion]
+
+    @override
+    def stop_sequences(self) -> list[str]:
+        return ["Question:"]
+
+    @override
+    def max_tokens(self) -> int | None:
+        return None
+
+    @override
+    def initial_prompt(self, subject_label: str) -> str | None:
+        return (
+            f"The following are multiple choice questions about {_humanized(subject_label)}. "
+            'Summarize your reasoning concisely, then conclude with "Therefore, the answer is: X", where X is '
+            "one of A, B, C, or D."
+        )
+
+    @override
+    def samples(self, item: dict[str, Any]) -> list[SampleBody]:
+        fields = self._reader.read(item)
+        keys = get_n_letters(len(fields.choices))
+        options = "\n".join(f"{key}. {choice}" for key, choice in zip(keys, fields.choices))
+        return [
+            SampleBody(
+                prompt=f"Question: {fields.raw_question}\n{options}",
+                cue="",  # no assistant cue — the model continues with its reasoning
+                possible_completions=[],  # free-form generation (normalized to None by ComposedEval)
+                ground_truth=keys[fields.correct_index],  # the bare answer letter
+            )
+        ]
+
+    @override
+    def extract_answer(
+        self,
+        completion_text: str,
+        *,
+        context: BaseMetricContext | list[BaseMetricContext] | None,
+        ground_truth: str | list[str] | None,
+        messages: list[Message],
+    ) -> str:
+        for stop in self.stop_sequences():
+            completion_text = completion_text.split(stop)[0]
+        match = self._answer_re.search(completion_text)
+        return match.group(1) if match else "[invalid]"
+
+
+def _mmlu_dataset(dataset: DatasetPolicy | None) -> DatasetPolicy:
+    return dataset if dataset is not None else pinned_by_framework("cais/mmlu")
+
+
+def _mmlu_choice(
+    id: str, styler: TaskStyler, dataset: DatasetPolicy | None = None, display_name: str | None = None
+) -> Benchmark:
+    # The loglikelihood variants differ only in their styler; MmluReader and the dev fewshot split are shared.
+    return ComposedBenchmark.choice(
+        id=id,
+        display_name=display_name,
+        reader=MmluReader(),
+        styler=styler,
+        sample_split="test",
+        fewshot_split="dev",
+        subjects=ListOfSubjects(MMLU_SUBJECTS),
+        dataset_policy=_mmlu_dataset(dataset),
+        language=Language.ENG,
+    )
+
+
+def mmlu(dataset: DatasetPolicy | None = None) -> Benchmark:
+    styler = MCStyle(question_prefix="Question: ", cue_text="Answer:", initial_prompt=_mc_preamble)
+    return _mmlu_choice("MMLU", styler, dataset)
+
+
+def mmlu_olmes(dataset: DatasetPolicy | None = None) -> Benchmark:
+    styler = MCStyle(
+        question_prefix="Question: ", cue_text="Answer:", space_prefixed_labels=True, initial_prompt=_mc_preamble
+    )
+    return _mmlu_choice("MMLU_OLMES", styler, dataset)
+
+
+def mmlu_full_text(dataset: DatasetPolicy | None = None) -> Benchmark:
+    styler = _FullTextMmluStyle(question_prefix="Question: ", cue_text="Answer:", initial_prompt=_full_text_preamble)
+    # The registry/hash identity is the compact "FullTextMMLU"; the display name keeps the spelled-out form.
+    return _mmlu_choice("FullTextMMLU", styler, dataset, display_name="Full Text MMLU")
+
+
+def mmlu_idk(dataset: DatasetPolicy | None = None) -> Benchmark:
+    styler = MCStyle(
+        question_prefix="Question: ", cue_text="Answer:", initial_prompt=_idk_preamble
+    ).with_abstention_option(" ?")
+    return _mmlu_choice("MMLU_IDK", styler, dataset)
+
+
+def mmlu_cot(dataset: DatasetPolicy | None = None) -> Benchmark:
+    # Free-form (completion) and 0-shot only, so it takes the general compose path with its own kind.
+    return ComposedBenchmark.compose(
+        id="MMLU_COT",
+        kind=_MmluCotKind(),
+        sample_split="test",
+        fewshot=NoFewShot(),
+        subjects=ListOfSubjects(MMLU_SUBJECTS),
+        dataset_policy=_mmlu_dataset(dataset),
+        language=Language.ENG,
+    )
+
+
+MMLU_BENCHMARKS: list[Benchmark] = [mmlu(), mmlu_olmes(), mmlu_full_text(), mmlu_idk(), mmlu_cot()]

{eval_framework-0.12.0 → eval_framework-0.13.0}/src/eval_framework/benchmarks/piqa.py

@@ -12,7 +12,6 @@ from typing import Any, final, override
 from eval_framework.choices import ChoiceFields, ChoiceReader
 from eval_framework.composed import ComposedBenchmark
 from eval_framework.contract import Benchmark
-from eval_framework.eval_kind import Choice
 from eval_framework.tasks.base import Language
 from eval_framework.tasks.dataset_loading import DatasetPolicy
 from eval_framework.tasks.dataset_revisions import pinned_frozen
@@ -46,11 +45,11 @@ def _piqa_benchmark(
     fewshot_split: str,
     dataset: DatasetPolicy | None = None,
 ) -> Benchmark:
-    kind = Choice(reader=PiqaReader(), styler=styler)
     dataset_policy = dataset if dataset is not None else pinned_frozen("ybisk/piqa")
-    return ComposedBenchmark.compose(
+    return ComposedBenchmark.choice(
         id=id,
-        kind=kind,
+        reader=PiqaReader(),
+        styler=styler,
         sample_split=sample_split,
         fewshot_split=fewshot_split,
         dataset_policy=dataset_policy,

{eval_framework-0.12.0 → eval_framework-0.13.0}/src/eval_framework/benchmarks/piqa_ellamind.py

@@ -10,7 +10,6 @@ from typing import Any, Literal, final, override
 from eval_framework.choices import ChoiceFields, ChoiceReader
 from eval_framework.composed import ComposedBenchmark
 from eval_framework.contract import Benchmark
-from eval_framework.eval_kind import Choice
 from eval_framework.subjects import ListOfSubjects
 from eval_framework.tasks.base import Language
 from eval_framework.tasks.dataset_loading import DatasetPolicy
@@ -43,11 +42,11 @@ _CUE_TEXT = "Antwort:"
 def _piqa_ellamind_benchmark(
     id: str, styler: TaskStyler, distractor_level: Literal["easy", "hard"], dataset: DatasetPolicy | None = None
 ) -> Benchmark:
-    kind = Choice(reader=PiqaReader(distractor_level), styler=styler)
     dataset_policy = dataset if dataset is not None else pinned_by_framework("ellamind/piqa-multilingual")
-    return ComposedBenchmark.compose(
+    return ComposedBenchmark.choice(
         id=id,
-        kind=kind,
+        reader=PiqaReader(distractor_level),
+        styler=styler,
         sample_split="validation",
         fewshot_split="validation",
         subjects=ListOfSubjects(["deu"]),