eval-framework 0.11.16__tar.gz → 0.12.0__tar.gz

{eval_framework-0.11.16 → eval_framework-0.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.11.16
+Version: 0.12.0
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -232,7 +232,7 @@ Requires-Dist: jsonlines>=4,<5
 Requires-Dist: lxml>=6.1.3,<7
 Requires-Dist: python-iso639>=2026.7.23
 Requires-Dist: wandb>=0.30.0,<1
-Requires-Dist: boto3>=1.43.92,<2
+Requires-Dist: boto3>=1.43.93,<2
 Requires-Dist: numpy>=2.5.3
 Requires-Dist: antlr4-python3-runtime==4.11.0
 Requires-Dist: scipy>=1.18.1,<2

{eval_framework-0.11.16 → eval_framework-0.12.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.11.16"
+version = "0.12.0"
 description = "Evaluation Framework"
 readme = "README.md"
 requires-python = ">=3.12,<3.15"
@@ -35,7 +35,7 @@ dependencies = [
     "lxml>=6.1.3,<7",
     "python-iso639>=2026.7.23",
     "wandb>=0.30.0,<1",
-    "boto3>=1.43.92,<2",
+    "boto3>=1.43.93,<2",
     "numpy>=2.5.3",
     "antlr4-python3-runtime==4.11.0",
     "scipy>=1.18.1,<2",

{eval_framework-0.11.16 → eval_framework-0.12.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.11.16"
+version = "0.12.0"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -39,7 +39,7 @@ dependencies = [
   "lxml>=6.1.3,<7",
   "python-iso639>=2026.7.23",
   "wandb>=0.30.0,<1",
-  "boto3>=1.43.92,<2",
+  "boto3>=1.43.93,<2",
   "numpy>=2.5.3",
   # is a dependency of sympy, but not explicitly listed in the requirements.txt
   # https://github.com/sympy/sympy/blob/0204fa34e8f6f6f8ccb4de01209be9a2345c9d6e/doc/src/contributing/dependencies.md?plain=1#L125

{eval_framework-0.11.16 → eval_framework-0.12.0}/src/eval_framework/benchmarks/goldenswag.py

@@ -11,7 +11,7 @@ from eval_framework.eval_kind import Choice
 from eval_framework.tasks.base import Language
 from eval_framework.tasks.dataset_loading import DatasetPolicy
 from eval_framework.tasks.dataset_revisions import pinned_by_framework
-from eval_framework.tasks.task_style import ClozeStyle, IdkClozeStyle, TaskStyler
+from eval_framework.tasks.task_style import ClozeStyle, TaskStyler
 
 _IDK_PREAMBLE = (
     "Complete the sentence only if you are confident, since mistakes may be penalised, while correct "
@@ -39,8 +39,10 @@ def goldenswag(dataset: DatasetPolicy | None = None) -> Benchmark:
 
 
 def goldenswag_idk(dataset: DatasetPolicy | None = None) -> Benchmark:
-    cloze = ClozeStyle(question_prefix="", cue_text="", trailing_newline=False)
-    styler = IdkClozeStyle(abstention_option=" I do not know.", initial_prompt=_IDK_PREAMBLE, cloze=cloze)
+    cloze = ClozeStyle(
+        question_prefix="", cue_text="", trailing_newline=False, initial_prompt=lambda _subject: _IDK_PREAMBLE
+    )
+    styler = cloze.with_abstention_option(" I do not know.")
     return _goldenswag_benchmark("GoldenSwag_IDK", styler, dataset)
 
 

eval_framework-0.12.0/src/eval_framework/benchmarks/mmlu.py

@@ -0,0 +1,116 @@
+"""MMLU: https://huggingface.co/datasets/cais/mmlu
+
+Multiple-choice knowledge questions across 57 subjects (each an HF config, loaded per subject); every
+prompt is prefaced by a subject-templated preamble. The composed variants:
+
+* ``MMLU`` / ``MMLU_OLMES`` — score the letter labels; OLMES adds a space before each option label.
+* ``Full Text MMLU`` — shows the options as a bulleted list and scores the full answer text.
+* ``MMLU_IDK`` — lets the model abstain with ``"?"`` and reports confidence-aware metrics.
+
+``MMLU_COT`` stays BaseTask until the composed design supports generative (completion) tasks.
+"""
+
+from typing import Any, final, override
+
+from eval_framework.choices import ChoiceFields, ChoiceReader
+from eval_framework.composed import ComposedBenchmark
+from eval_framework.contract import Benchmark
+from eval_framework.eval_kind import Choice
+from eval_framework.subjects import ListOfSubjects
+from eval_framework.tasks.base import Language
+from eval_framework.tasks.benchmarks.mmlu import MMLU_SUBJECTS  # single source of truth for the 57 subjects
+from eval_framework.tasks.dataset_loading import DatasetPolicy
+from eval_framework.tasks.dataset_revisions import pinned_by_framework
+from eval_framework.tasks.task_style import ClozeStyle, MCStyle, TaskStyler
+
+
+@final
+class MmluReader(ChoiceReader):
+    """Reads an MMLU item: the question and its four answer choices, with the correct one at ``answer``."""
+
+    @override
+    def read(self, item: dict[str, Any]) -> ChoiceFields:
+        return ChoiceFields(
+            raw_question=item["question"].strip(),
+            choices=item["choices"],
+            correct_index=item["answer"],
+        )
+
+
+@final
+class _FullTextMmluStyle(ClozeStyle):
+    """Cloze scoring (full answer text) that additionally shows the options as a bulleted list."""
+
+    @override
+    def get_instruction_text(self, raw_question: str, choices: list[str]) -> str:
+        options = "".join(f"- {choice}\n" for choice in choices)
+        return f"{self.question_prefix}{raw_question}\nPossible answers:\n{options}"
+
+
+def _humanized(subject_label: str) -> str:
+    # Subject labels are underscored config names ("abstract_algebra"); read them as prose.
+    return " ".join(subject_label.split("_"))
+
+
+def _mc_preamble(subject_label: str) -> str:
+    return f"The following are multiple choice questions (with answers) about {_humanized(subject_label)}."
+
+
+def _full_text_preamble(subject_label: str) -> str:
+    return (
+        f"The following are multiple choice questions (with possible answers) about {_humanized(subject_label)}.\n"
+        "Answer with the full text of the correct answer."
+    )
+
+
+def _idk_preamble(subject_label: str) -> str:
+    # Unlike the other variants, the IDK preamble names the subject by its raw (underscored) key.
+    return (
+        f"The following are multiple choice questions (with answers) about {subject_label}. "
+        "Answer only if you are confident, since mistakes may be penalised, while correct answers receive points. "
+        "It is acceptable to answer with '?' if you are unsure, and you will receive 0 points."
+    )
+
+
+def _mmlu_benchmark(
+    id: str, styler: TaskStyler, dataset: DatasetPolicy | None = None, display_name: str | None = None
+) -> Benchmark:
+    dataset_policy = dataset if dataset is not None else pinned_by_framework("cais/mmlu")
+    return ComposedBenchmark.compose(
+        id=id,
+        display_name=display_name,
+        kind=Choice(reader=MmluReader(), styler=styler),
+        sample_split="test",
+        fewshot_split="dev",
+        subjects=ListOfSubjects(MMLU_SUBJECTS),
+        dataset_policy=dataset_policy,
+        language=Language.ENG,
+    )
+
+
+def mmlu(dataset: DatasetPolicy | None = None) -> Benchmark:
+    styler = MCStyle(question_prefix="Question: ", cue_text="Answer:", initial_prompt=_mc_preamble)
+    return _mmlu_benchmark("MMLU", styler, dataset)
+
+
+def mmlu_olmes(dataset: DatasetPolicy | None = None) -> Benchmark:
+    styler = MCStyle(
+        question_prefix="Question: ", cue_text="Answer:", space_prefixed_labels=True, initial_prompt=_mc_preamble
+    )
+    return _mmlu_benchmark("MMLU_OLMES", styler, dataset)
+
+
+def mmlu_full_text(dataset: DatasetPolicy | None = None) -> Benchmark:
+    styler = _FullTextMmluStyle(question_prefix="Question: ", cue_text="Answer:", initial_prompt=_full_text_preamble)
+    # The registry/hash identity is the compact "FullTextMMLU"; the display name keeps the spelled-out form.
+    return _mmlu_benchmark("FullTextMMLU", styler, dataset, display_name="Full Text MMLU")
+
+
+def mmlu_idk(dataset: DatasetPolicy | None = None) -> Benchmark:
+    styler = MCStyle(
+        question_prefix="Question: ", cue_text="Answer:", initial_prompt=_idk_preamble
+    ).with_abstention_option(" ?")
+    return _mmlu_benchmark("MMLU_IDK", styler, dataset)
+
+
+MMLU_BENCHMARKS: list[Benchmark] = [mmlu(), mmlu_olmes(), mmlu_full_text(), mmlu_idk()]

{eval_framework-0.11.16 → eval_framework-0.12.0}/src/eval_framework/benchmarks/piqa.py

@@ -16,7 +16,7 @@ from eval_framework.eval_kind import Choice
 from eval_framework.tasks.base import Language
 from eval_framework.tasks.dataset_loading import DatasetPolicy
 from eval_framework.tasks.dataset_revisions import pinned_frozen
-from eval_framework.tasks.task_style import ClozeStyle, IdkClozeStyle, MCStyle, TaskStyler
+from eval_framework.tasks.task_style import ClozeStyle, MCStyle, TaskStyler
 
 _IDK_PREAMBLE = (
     "Complete the sentence only if you are confident, since mistakes may be penalised, while correct "
@@ -75,7 +75,7 @@ def piqa_olmes(dataset: DatasetPolicy | None = None) -> Benchmark:
 def piqa_idk(dataset: DatasetPolicy | None = None) -> Benchmark:
     return _piqa_benchmark(
         "PIQA_IDK",
-        IdkClozeStyle(abstention_option=" I do not know", initial_prompt=_IDK_PREAMBLE),
+        ClozeStyle(initial_prompt=lambda _subject: _IDK_PREAMBLE).with_abstention_option(" I do not know"),
         sample_split="validation",
         fewshot_split="test",
         dataset=dataset,

{eval_framework-0.11.16 → eval_framework-0.12.0}/src/eval_framework/composed.py

@@ -15,6 +15,7 @@ from eval_framework.metrics.efficiency.bytes_per_sequence_position import (
     SequencePositionsCompletion,
     SequencePositionsLoglikelihood,
 )
+from eval_framework.metrics.efficiency.finish_reason import FinishReason
 from eval_framework.metrics.efficiency.token_counters import TokenCounts
 from eval_framework.shared.errors import raise_errors
 from eval_framework.shared.types import Completion, Error, RawCompletion
@@ -85,6 +86,7 @@ class ComposedEval(Eval):
             dataset = self._load_dataset(subject.load_key)
             fewshot_pool = dataset[self.fewshot_split] if self.num_fewshot > 0 else []
             assert len(dataset[self.sample_split]) > 0
+            initial_prompt = self._kind.initial_prompt(subject.label)
             sample_id = 0  # ids and the num_samples cap are per subject, matching BaseTask
             done = False
             for item in dataset[self.sample_split]:
@@ -96,7 +98,7 @@ class ComposedEval(Eval):
                     yield Sample(
                         id=sample_id,
                         subject=subject.label,
-                        messages=self._messages(prefix, sample_body),
+                        messages=self._messages(prefix, sample_body, initial_prompt),
                         ground_truth=sample_body.ground_truth,
                         possible_completions=sample_body.possible_completions,
                         context=None,
@@ -106,9 +108,8 @@ class ComposedEval(Eval):
                         done = True
                         break
 
-    def _messages(self, prefix: list[Message], body: SampleBody) -> list[Message]:
+    def _messages(self, prefix: list[Message], body: SampleBody, initial_prompt: str | None) -> list[Message]:
         messages = [*prefix, Message(role=Role.USER, content=body.prompt)]
-        initial_prompt = self._kind.initial_prompt()
         if initial_prompt is not None:
             first = messages[0]
             messages[0] = Message(role=first.role, content=f"{initial_prompt}\n\n{first.content}")
@@ -236,6 +237,14 @@ class ComposedEval(Eval):
             )
         return completion_list
 
+    @override
+    def get_stop_sequences(self) -> list[str]:
+        return []
+
+    @override
+    def get_max_tokens(self) -> int | None:
+        return None
+
     @override
     def get_response_type(self) -> ResponseType:
         return self._kind.response_type
@@ -250,7 +259,7 @@ def _metrics_for(kind: EvalKind) -> list[type["BaseMetric"]]:
     response_type_metrics: list[type[BaseMetric]]
     match kind.response_type:
         case ResponseType.COMPLETION:
-            response_type_metrics = [BytesCompletion, SequencePositionsCompletion, TokenCounts]
+            response_type_metrics = [BytesCompletion, SequencePositionsCompletion, TokenCounts, FinishReason]
         case ResponseType.LOGLIKELIHOODS:
             response_type_metrics = [BytesLoglikelihood, SequencePositionsLoglikelihood]
         case _:

{eval_framework-0.11.16 → eval_framework-0.12.0}/src/eval_framework/contract.py

@@ -52,6 +52,14 @@ class Eval(ABC):
     def get_metadata(self) -> dict[str, str | list[str]]:
         """Descriptive metadata about the eval for result reporting."""
 
+    @abstractmethod
+    def get_stop_sequences(self) -> list[str]:
+        """Stop sequences the eval requests for completion generation."""
+
+    @abstractmethod
+    def get_max_tokens(self) -> int | None:
+        """Token limit the eval requests for completion generation, or ``None`` for no limit."""
+
     @abstractmethod
     def get_response_type(self) -> ResponseType: ...
 

{eval_framework-0.11.16 → eval_framework-0.12.0}/src/eval_framework/eval_kind.py

@@ -47,8 +47,9 @@ class EvalKind(ABC):
         """Kind-specific metadata merged into the eval's ``get_metadata`` (e.g. the task style)."""
         return {}
 
-    def initial_prompt(self) -> str | None:
-        """A preamble prepended once at the top of the prompt (before any few-shot examples), or None."""
+    def initial_prompt(self, subject_label: str) -> str | None:
+        """A preamble prepended once at the top of the prompt for the given subject (before any few-shot
+        examples), or None."""
         return None
 
 
@@ -90,5 +91,5 @@ class Choice(EvalKind):
         return self._styler.get_extra_metadata()
 
     @override
-    def initial_prompt(self) -> str | None:
-        return self._styler.initial_prompt()
+    def initial_prompt(self, subject_label: str) -> str | None:
+        return self._styler.initial_prompt(subject_label)

eval_framework-0.12.0/src/eval_framework/metrics/efficiency/finish_reason.py

@@ -0,0 +1,39 @@
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+
+class FinishReason(BaseMetric[Completion]):
+    """Why the backend ended the generation, as one 0/1 indicator per reason.
+
+    Averaged over a benchmark, each key becomes a rate: `FinishReason/Repetition`
+    is the share of completions vLLM aborted through its `repetition_detection`
+    sampling parameter (looping output), `FinishReason/Length` the share that hit
+    `max_tokens`, and `FinishReason/Stop` the share that ended on their own (EOS
+    or a stop sequence). Reasons outside these keys (e.g. `tool_calls`) count as
+    0 for all of them.
+
+    Reads the `finish_reason` backends already attach to the response. All values
+    are None when the backend did not report one (e.g. HuggingFace) or when the
+    sample errored, so such samples do not dilute the rates.
+    """
+
+    NAME = "FinishReason"
+    KEYS = ["Stop", "Length", "Repetition"]
+    _REASONS = {"Stop": "stop", "Length": "length", "Repetition": "repetition"}
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        finish_reason = response.raw_completion_finish_reason
+        if response.error or finish_reason is None:
+            values: dict[str, float | None] = {key: None for key in self.KEYS}
+        else:
+            values = {key: float(finish_reason.lower() == reason) for key, reason in self._REASONS.items()}
+
+        return [
+            MetricResult(
+                metric_name=f"{self.NAME}/{key}",
+                value=value,
+                higher_is_better=key == "Stop",
+                error=response.error,
+            )
+            for key, value in values.items()
+        ]

{eval_framework-0.11.16 → eval_framework-0.12.0}/src/eval_framework/response_generator.py

@@ -80,8 +80,8 @@ class ResponseGenerator:
         """
         llm_stop_sequences = getattr(self.llm, "stop_sequences", None)
         llm_max_tokens = getattr(self.llm, "max_tokens", None)
-        task_stop_sequences = getattr(self.task, "stop_sequences", None)
-        task_max_tokens = self.config.max_tokens or getattr(self.task, "max_tokens", None)
+        task_stop_sequences = self.task.get_stop_sequences()
+        task_max_tokens = self.config.max_tokens or self.task.get_max_tokens()
         # if both task and model define a max_token, the smaller value is used
         max_tokens = min(
             [x for x in [llm_max_tokens, task_max_tokens] if x is not None],
@@ -89,7 +89,7 @@ class ResponseGenerator:
         )
         logger.info(f"Set max_tokens to {max_tokens}")
         # if both task and model define stop sequences, those are merged into one list
-        stop_sequences_merged = (llm_stop_sequences or []) + (task_stop_sequences or [])
+        stop_sequences_merged = (llm_stop_sequences or []) + task_stop_sequences
         stop_sequences = sorted(list(set(stop_sequences_merged))) if stop_sequences_merged else None
         logger.info(f"Set stop_sequences to {stop_sequences}")
         return stop_sequences, max_tokens

{eval_framework-0.11.16 → eval_framework-0.12.0}/src/eval_framework/tasks/base.py

@@ -18,6 +18,7 @@ from eval_framework.metrics.efficiency.bytes_per_sequence_position import (
     SequencePositionsCompletion,
     SequencePositionsLoglikelihood,
 )
+from eval_framework.metrics.efficiency.finish_reason import FinishReason
 from eval_framework.metrics.efficiency.token_counters import TokenCounts
 from eval_framework.shared.errors import raise_errors
 from eval_framework.shared.types import BaseMetricContext, Completion, Error, RawCompletion
@@ -444,6 +445,12 @@ class BaseTask[SubjectType](Eval):
             )
         return completion_list
 
+    def get_stop_sequences(self) -> list[str]:
+        return self.stop_sequences or []
+
+    def get_max_tokens(self) -> int | None:
+        return self.max_tokens
+
     @classmethod
     def get_response_type(cls) -> ResponseType:
         """Return the response type of the task (or the styler if it exists)."""
@@ -468,6 +475,7 @@ class BaseTask[SubjectType](Eval):
                     BytesCompletion,
                     SequencePositionsCompletion,
                     TokenCounts,
+                    FinishReason,
                 ]
             case ResponseType.LOGLIKELIHOODS:
                 metrics = [BytesLoglikelihood, SequencePositionsLoglikelihood]

{eval_framework-0.11.16 → eval_framework-0.12.0}/src/eval_framework/tasks/benchmarks/mmlu.py

@@ -8,9 +8,6 @@ from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
     AccuracyNormLoglikelihood,
 )
 from eval_framework.metrics.loglikelihood.bits_per_byte import BitsPerByteLoglikelihood
-from eval_framework.metrics.loglikelihood.confidence_weighted_accuracy import ConfidenceWeightedAccuracy
-from eval_framework.metrics.loglikelihood.dcs import DistributionalCorrectnessScore
-from eval_framework.metrics.loglikelihood.ternary import TernaryScore
 from eval_framework.tasks.base import BaseTask, Language, ResponseType, Sample
 from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 from eval_framework.tasks.utils import get_n_letters
@@ -126,73 +123,6 @@ class MMLU(BaseTask[str]):
         return [f" {key}" for key in self.keys]
 
 
-class MMLU_OLMES(MMLU):
-    """
-    MMLU with OLMES-style prompt: space before each label in the prompt (" A.", " B.", ...).
-    """
-
-    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
-
-    NAME = "MMLU_OLMES"
-
-    def _get_instruction_text(self, item: dict[str, Any]) -> str:
-        question = item["question"].strip()
-        choices = "".join([f" {key}. {choice}\n" for key, choice in zip(self.keys, item["choices"])])
-        return f"Question: {question}\n{choices}"
-
-
-class FullTextMMLU(MMLU):
-    """MMLU dataset but where the model is expected to replicate choice text, rather than just the key."""
-
-    NAME = "Full Text MMLU"
-    METRICS = [
-        AccuracyLoglikelihood,
-        AccuracyNormLoglikelihood,
-        AccuracyBayesianLoglikelihood,
-        BitsPerByteLoglikelihood,
-    ]
-
-    def _get_initial_prompt_text(self, item: dict[str, Any]) -> str:
-        subject_name = self._get_subject_name(item)
-        return f"""The following are multiple choice questions (with possible answers) about {subject_name}.
-Answer with the full text of the correct answer."""
-
-    def _get_instruction_text(self, item: dict[str, Any]) -> str:
-        question = item["question"].strip()
-        choices = "".join([f"- {choice}\n" for choice in item["choices"]])
-        return f"Question: {question}\nPossible answers:\n{choices}"
-
-    def _get_ground_truth(self, item: dict[str, Any]) -> str | None:
-        return f" {item['choices'][item['answer']]}"
-
-    def _get_possible_completions(self, item: dict[str, Any]) -> list[str] | None:
-        return [f" {choice}" for choice in item["choices"]]
-
-
-class MMLU_IDK(MMLU):
-    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
-    NAME = "MMLU_IDK"
-    METRICS = [
-        AccuracyLoglikelihood,
-        AccuracyNormLoglikelihood,
-        AccuracyBayesianLoglikelihood,
-        ConfidenceWeightedAccuracy,
-        DistributionalCorrectnessScore,
-        TernaryScore,
-    ]
-
-    def _get_initial_prompt_text(self, item: dict[str, Any]) -> str:
-        return (
-            f"The following are multiple choice questions (with answers) about {item['subject']}. "
-            "Answer only if you are confident, since mistakes may be penalised, while correct answers receive points. "
-            "It is acceptable to answer with '?' if you are unsure, and you will receive 0 points."
-        )
-
-    def _get_possible_completions(self, item: dict[str, Any]) -> list[str] | None:
-        completions = super()._get_possible_completions(item)
-        return (completions or []) + [" ?"]
-
-
 class MMLU_COT(MMLU):
     """
     MMLU dataset with instruction to summarize reasoning and conclude with answer.

{eval_framework-0.11.16 → eval_framework-0.12.0}/src/eval_framework/tasks/task_names.py

@@ -7,6 +7,7 @@ from eval_framework.benchmarks.gpqa_ellamind import GPQA_ELLAMIND_BENCHMARKS
 from eval_framework.benchmarks.hellaswag import HELLASWAG_BENCHMARKS
 from eval_framework.benchmarks.hellaswag_ellamind import HELLASWAG_ELLAMIND_BENCHMARKS
 from eval_framework.benchmarks.hle_ellamind import HLE_ELLAMIND_BENCHMARKS
+from eval_framework.benchmarks.mmlu import MMLU_BENCHMARKS
 from eval_framework.benchmarks.piqa import PIQA_BENCHMARKS
 from eval_framework.benchmarks.piqa_ellamind import PIQA_ELLAMIND_BENCHMARKS
 from eval_framework.benchmarks.simpleqa_ellamind import SIMPLEQA_ELLAMIND_BENCHMARKS
@@ -117,10 +118,9 @@ def register_math_reasoning_tasks(registry: Registry) -> None:
 
 def register_mmlu_tasks(registry: Registry) -> None:
     """Register mmlu benchmark tasks."""
-    register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU", registry=registry)
-    register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU_IDK", registry=registry)
-    register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU_OLMES", registry=registry)
-    register_lazy_task("eval_framework.tasks.benchmarks.mmlu.FullTextMMLU", registry=registry)
+    for benchmark in MMLU_BENCHMARKS:  # composed: MMLU, MMLU_OLMES, FullTextMMLU, MMLU_IDK
+        registry.add(benchmark)
+    # MMLU_COT is a generative (completion) task, not yet supported by the composed design.
     register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU_COT", registry=registry)