eval-framework 0.9.0__tar.gz → 0.9.2__tar.gz

{eval_framework-0.9.0 → eval_framework-0.9.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.9.0
+Version: 0.9.2
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -216,7 +216,7 @@ Requires-Dist: xmltodict>=1.0.4,<1.1
 Requires-Dist: pydantic>=2.13.4,<3
 Requires-Dist: datasets>=5.0.1,<6
 Requires-Dist: pycountry>=26.2.16,<27
-Requires-Dist: nltk>=3.10.1,<4
+Requires-Dist: nltk>=3.10.2,<4
 Requires-Dist: python-dotenv>=1.2.2,<2
 Requires-Dist: lingua-language-detector>=2.2.0,<3
 Requires-Dist: google-crc32c>=1.8.0,<2
@@ -450,11 +450,13 @@ from eval_framework.main import main
 from eval_framework.tasks.eval_config import EvalConfig
 from template_formatting.formatter import HFFormatter
 
+
 # Define your model
 class MyHuggingFaceModel(HFLLM):
     LLM_NAME = "microsoft/DialoGPT-medium"
     DEFAULT_FORMATTER = partial(HFFormatter, "microsoft/DialoGPT-medium")
 
+
 if __name__ == "__main__":
     # Initialize your model
     llm = MyHuggingFaceModel()

{eval_framework-0.9.0 → eval_framework-0.9.2}/README.md

@@ -189,11 +189,13 @@ from eval_framework.main import main
 from eval_framework.tasks.eval_config import EvalConfig
 from template_formatting.formatter import HFFormatter
 
+
 # Define your model
 class MyHuggingFaceModel(HFLLM):
     LLM_NAME = "microsoft/DialoGPT-medium"
     DEFAULT_FORMATTER = partial(HFFormatter, "microsoft/DialoGPT-medium")
 
+
 if __name__ == "__main__":
     # Initialize your model
     llm = MyHuggingFaceModel()

{eval_framework-0.9.0 → eval_framework-0.9.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.9.0"
+version = "0.9.2"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -23,7 +23,7 @@ dependencies = [
   "pydantic>=2.13.4,<3",
   "datasets>=5.0.1,<6",
   "pycountry>=26.2.16,<27",
-  "nltk>=3.10.1,<4",
+  "nltk>=3.10.2,<4",
   "python-dotenv>=1.2.2,<2",
   "lingua-language-detector>=2.2.0,<3",
   "google-crc32c>=1.8.0,<2",

{eval_framework-0.9.0 → eval_framework-0.9.2}/src/eval_framework/metrics/completion/code_assertion.py

@@ -2,7 +2,7 @@ from llm_sandbox.exceptions import SandboxTimeoutError
 
 from eval_framework.metrics.base import BaseMetric, MetricResult
 from eval_framework.shared.types import Completion
-from eval_framework.tasks.utils import run_python_code
+from eval_framework.tasks.utils import DockerReturnedEmptyOutput, run_python_code
 
 
 class CodeCompletionAssertion(BaseMetric[Completion]):
@@ -16,7 +16,7 @@ class CodeCompletionAssertion(BaseMetric[Completion]):
         code = response.completion
         try:
             output = run_python_code(code, image="python:3.12-slim", runtime_configs={"mem_limit": "512m"})
-        except SandboxTimeoutError:
+        except (SandboxTimeoutError, DockerReturnedEmptyOutput):
             # The submitted code timed out (e.g. an infinite loop) -- a failing sample, not an infra
             # problem.
             import traceback

{eval_framework-0.9.0 → eval_framework-0.9.2}/src/eval_framework/metrics/completion/minerva_math_utils.py

@@ -350,8 +350,17 @@ def strip_string_hendrycks(string: str) -> str:
     return string
 
 
+# MATH gold answers stay under ~80 chars.
+_PLAUSIBLE_ANSWER_LEN = 128
+
+
 def is_equiv_minerva(x1: str, x2: str, timeout_seconds: int = 5) -> bool:
     """Sympy-based equivalence (Minerva)."""
+    # extract_answers can fall back to the whole completion; sympy grinds on such
+    # 1000+ char strings until the timeout. A side far longer than the other can't
+    # be an equivalent final answer, so refuse it before parsing.
+    if max(len(x1), len(x2)) > max(_PLAUSIBLE_ANSWER_LEN, 3 * min(len(x1), len(x2))):
+        return False
 
     def _timeout_handler(signum: Any, frame: Any) -> None:
         raise TimeoutError()

{eval_framework-0.9.0 → eval_framework-0.9.2}/src/eval_framework/response_generator.py

@@ -26,7 +26,7 @@ from eval_framework.shared.types import (
     Loglikelihood,
     RawLoglikelihood,
 )
-from eval_framework.tasks.base import Language, ResponseType, Sample
+from eval_framework.tasks.base import RANDOM_SEED, Language, ResponseType, Sample
 from eval_framework.tasks.eval_config import EvalConfig
 from eval_framework.tasks.utils import raise_errors
 from eval_framework.utils.constants import RED, RESET
@@ -67,6 +67,7 @@ class ResponseGenerator:
                 config.task_subjects,
                 config.hf_revision,
                 user_prompt_suffix=config.user_prompt_suffix,
+                seed=RANDOM_SEED,
             )
         else:
             self.task = registry()[config.task_name].create(
@@ -74,6 +75,7 @@ class ResponseGenerator:
                 config.task_subjects,
                 config.hf_revision,
                 user_prompt_suffix=config.user_prompt_suffix,
+                seed=RANDOM_SEED,
             )
 
         self.response_type = self.task.get_response_type()

{eval_framework-0.9.0 → eval_framework-0.9.2}/src/eval_framework/tasks/base.py

@@ -147,6 +147,7 @@ class BaseTask[SubjectType](Task):
         self.stop_sequences: list[str] | None = None
         self.max_tokens: int | None = None
         self.hf_revision: str | None = self._apply_hf_revision()
+        self.rnd: random.Random | None = None
 
     def _apply_hf_revision(self, custom_hf_revision: str | None = None) -> str | None:
         # Precedence: CLI/config override > REVISION_LOCKFILE pin.
@@ -167,12 +168,13 @@ class BaseTask[SubjectType](Task):
         custom_subjects: list[str] | None,
         custom_hf_revision: str | None,
         user_prompt_suffix: str | None = None,
+        seed: int | None = RANDOM_SEED,
     ) -> Self:
         instance = cls(num_fewshot=num_fewshot)
         if user_prompt_suffix is not None and instance.get_response_type() != ResponseType.COMPLETION:
             raise ValueError("user_prompt_suffix is only supported for completion tasks.")
         instance.user_prompt_suffix = user_prompt_suffix
-
+        instance.rnd = random.Random(seed)
         # If custom subjects were provided during initialization, they take precedence over the class-level SUBJECTS.
         filtered_subjects = instance._filter_task_subjects(custom_subjects=custom_subjects)
         if filtered_subjects:
@@ -190,11 +192,27 @@ class BaseTask[SubjectType](Task):
 
         assert hasattr(self, "SUBJECTS") and len(self.SUBJECTS) > 0
         if isinstance(self.SUBJECTS[0], tuple):
-            # subjects are specified as strings but we need tuples
-            filters = [tuple(item.strip() for item in subject.split(",")) for subject in custom_subjects]
+            # subjects are specified as comma-separated strings but tuple positions may each hold a
+            # different type (e.g. tuple[str, int, str]). Infer the expected type per position from an
+            # actual subject and cast each part back to it, so it compares equal to the real values
+            # below instead of just their str() form. "*" is a wildcard sentinel and stays a string.
+            num_items = len(self.SUBJECTS[0])
+            position_types = [type(self.SUBJECTS[0][i]) for i in range(num_items)]
+
+            def cast(raw: str, i: int) -> Any:
+                raw = raw.strip()
+                return raw if raw == "*" else position_types[i](raw)
+
+            filters = []
+            for custom_subject in custom_subjects:
+                parts = custom_subject.split(",")
+                assert len(parts) == num_items, (
+                    f"Subject '{custom_subject}' has {len(parts)} parts, expected {num_items} for "
+                    f"task {self.display_name()}"
+                )
+                filters.append(tuple(cast(part, i) for i, part in enumerate(parts)))
 
             # check if all parts of custom subjects exists (* is a wildcard)
-            num_items = len(self.SUBJECTS[0])
             legal_values = [
                 set([s[i] for s in self.SUBJECTS if isinstance(s, tuple)] + ["*"]) for i in range(num_items)
             ]
@@ -204,18 +222,18 @@ class BaseTask[SubjectType](Task):
                     assert v in legal_values[i], f"Subject part {v} not found in task {self.__class__.__name__}"
 
             # filter task subjects. * is a supported wildcard for a specific item in a tuple, e.g. "DE_DE, *"
-            chosen_subjects = []
+            chosen_subjects: list[tuple] = []
             for subject in self.SUBJECTS:
                 subject_tuple = subject if isinstance(subject, tuple) else tuple(str(subject).split(","))
                 for filter in filters:
                     if all(filter[i] == "*" or filter[i] == subject_tuple[i] for i in range(num_items)):
                         chosen_subjects.append(subject_tuple)
                         break
-            return chosen_subjects  # type: ignore[return-value]
+            return chosen_subjects
         else:
             for cs in custom_subjects:
                 assert cs in self.SUBJECTS, f"Subject {cs} not found in task {self.__class__.__name__}"
-            return custom_subjects  # type: ignore[return-value]
+            return custom_subjects
 
     def _load_hf_dataset(self, **kwargs: Any) -> Any:
         cache_dir: str = os.environ.get("HF_DATASET_CACHE_DIR", f"{Path.home()}/.cache/huggingface/datasets")
@@ -228,8 +246,8 @@ class BaseTask[SubjectType](Task):
         )
 
     def _shuffle_splits(self, hf_dataset: DatasetDict) -> dict[str, Any]:
+        assert self.rnd is not None, "Task RNG is unseeded; build tasks via `with_overwrite`."
         dataset = {}
-        self.rnd = random.Random(RANDOM_SEED)
 
         for split, data in hf_dataset.items():
             if split not in [self.SAMPLE_SPLIT, self.FEWSHOT_SPLIT]:
@@ -397,6 +415,7 @@ class BaseTask[SubjectType](Task):
         return None
 
     def _sample_fewshot_examples(self, item: dict[str, Any]) -> list[dict]:
+        assert self.rnd is not None, "Task RNG is unseeded; build tasks via `with_overwrite`."
         if self.FEWSHOT_SPLIT == self.SAMPLE_SPLIT:
             # If the fewshot and sample splits are the same, we risk including the current eval item
             # as a fewshot example (leaking the answer). To prevent this, sample one extra example,

{eval_framework-0.9.0 → eval_framework-0.9.2}/src/eval_framework/tasks/benchmarks/mbpp.py

@@ -102,6 +102,7 @@ class MBPP(BaseTask[str]):
         return f"{BEGIN}\n" + target + f"\n{END}"
 
     def _sample_fewshot_examples(self, item: dict[str, Any]) -> list[dict]:
+        assert self.rnd is not None
         fewshot_examples = self.rnd.sample(self.dataset[self.FEWSHOT_SPLIT], self.num_fewshot)
         return fewshot_examples
 

{eval_framework-0.9.0 → eval_framework-0.9.2}/src/eval_framework/tasks/registry.py

@@ -5,7 +5,7 @@ from abc import ABC, abstractmethod
 from collections.abc import Generator, Iterator, Sequence
 from typing import TYPE_CHECKING, Any
 
-from eval_framework.tasks.base import BaseTask, ResponseType
+from eval_framework.tasks.base import RANDOM_SEED, BaseTask, ResponseType
 from eval_framework.tasks.perturbation import PerturbationConfig, create_perturbation_class
 from template_formatting.formatter import BaseFormatter
 
@@ -62,6 +62,7 @@ class EvalFactory(ABC):
         custom_subjects: list[str] | None,
         custom_hf_revision: str | None,
         user_prompt_suffix: str | None = None,
+        seed: int | None = None,
     ) -> BaseTask: ...
 
     @abstractmethod
@@ -72,6 +73,7 @@ class EvalFactory(ABC):
         custom_subjects: list[str] | None,
         custom_hf_revision: str | None,
         user_prompt_suffix: str | None = None,
+        seed: int | None = None,
     ) -> BaseTask: ...
 
     @abstractmethod
@@ -115,12 +117,14 @@ class _Lazy(EvalFactory):
         custom_subjects: list[str] | None,
         custom_hf_revision: str | None,
         user_prompt_suffix: str | None = None,
+        seed: int | None = None,
     ) -> BaseTask:
         return self.task_class().with_overwrite(
             num_fewshot=num_fewshot,
             custom_subjects=custom_subjects,
             custom_hf_revision=custom_hf_revision,
             user_prompt_suffix=user_prompt_suffix,
+            seed=seed,
         )
 
     def create_perturbation(
@@ -130,6 +134,7 @@ class _Lazy(EvalFactory):
         custom_subjects: list[str] | None,
         custom_hf_revision: str | None,
         user_prompt_suffix: str | None = None,
+        seed: int | None = None,
     ) -> BaseTask:
         perturbation_task_class = create_perturbation_class(self.task_class(), perturbation_config)
         return perturbation_task_class.with_overwrite(
@@ -137,6 +142,7 @@ class _Lazy(EvalFactory):
             custom_subjects=custom_subjects,
             custom_hf_revision=custom_hf_revision,
             user_prompt_suffix=user_prompt_suffix,
+            seed=seed,
         )
 
     def response_type(self) -> ResponseType:
@@ -153,9 +159,9 @@ class _Lazy(EvalFactory):
 
     def markdown_doc(self, formatters: Sequence[BaseFormatter]) -> str:
         try:
-            task = self.create(num_fewshot=1, custom_subjects=None, custom_hf_revision=None)
+            task = self.create(num_fewshot=1, custom_subjects=None, custom_hf_revision=None, seed=RANDOM_SEED)
         except (TypeError, ValueError, AssertionError):
-            task = self.create(num_fewshot=0, custom_subjects=None, custom_hf_revision=None)
+            task = self.create(num_fewshot=0, custom_subjects=None, custom_hf_revision=None, seed=RANDOM_SEED)
         return task.markdown_doc(formatters)
 
 
@@ -178,12 +184,14 @@ class _Eager(EvalFactory):
         custom_subjects: list[str] | None,
         custom_hf_revision: str | None,
         user_prompt_suffix: str | None = None,
+        seed: int | None = None,
     ) -> BaseTask:
         return self._task.with_overwrite(
             num_fewshot=num_fewshot,
             custom_subjects=custom_subjects,
             custom_hf_revision=custom_hf_revision,
             user_prompt_suffix=user_prompt_suffix,
+            seed=seed,
         )
 
     def create_perturbation(
@@ -193,6 +201,7 @@ class _Eager(EvalFactory):
         custom_subjects: list[str] | None,
         custom_hf_revision: str | None,
         user_prompt_suffix: str | None = None,
+        seed: int | None = None,
     ) -> BaseTask:
         perturbation_task_class = create_perturbation_class(self._task, perturbation_config)
         return perturbation_task_class.with_overwrite(
@@ -200,6 +209,7 @@ class _Eager(EvalFactory):
             custom_subjects=custom_subjects,
             custom_hf_revision=custom_hf_revision,
             user_prompt_suffix=user_prompt_suffix,
+            seed=seed,
         )
 
     def response_type(self) -> ResponseType:
@@ -216,27 +226,22 @@ class _Eager(EvalFactory):
 
     def markdown_doc(self, formatters: Sequence[BaseFormatter]) -> str:
         try:
-            task = self.create(num_fewshot=1, custom_subjects=None, custom_hf_revision=None)
+            task = self.create(num_fewshot=1, custom_subjects=None, custom_hf_revision=None, seed=RANDOM_SEED)
         except (TypeError, ValueError, AssertionError):
-            task = self.create(num_fewshot=0, custom_subjects=None, custom_hf_revision=None)
+            task = self.create(num_fewshot=0, custom_subjects=None, custom_hf_revision=None, seed=RANDOM_SEED)
         return task.markdown_doc(formatters)
 
 
 class Registry:
-    """A registry for tasks with support for lazy loading.
-
-    Task names are hashed based on the upper-case name, to avoid issues with
-    ambiguous naming.
-    """
+    """A registry for Tasks"""
 
     def __init__(self) -> None:
-        # TODO: Lookup only with upper names
-        self._registry: dict[str, tuple[str, EvalFactory]] = dict()
+        self._registry: dict[str, EvalFactory] = dict()
 
     def __iter__(self) -> Iterator[str]:
         """Iterate over all task names in the registry."""
-        for name, _ in self._registry.values():
-            yield name
+        for factory in self._registry.values():
+            yield factory.id()
 
     def task_names(self) -> list[str]:
         """The names of all registered tasks."""
@@ -244,7 +249,8 @@ class Registry:
 
     def items(self) -> Iterator[tuple[str, EvalFactory]]:
         """Iterate over `(task name, EvalFactory)` pairs in the registry."""
-        yield from self._registry.values()
+        for factory in self._registry.values():
+            yield factory.id(), factory
 
     @staticmethod
     def _task_key(name: str, /) -> str:
@@ -262,26 +268,25 @@ class Registry:
     def __getitem__(self, name: str, /) -> EvalFactory:
         task_key = self._task_key(name)
         try:
-            _, factory = self._registry[task_key]
+            return self._registry[task_key]
         except KeyError:
             raise KeyError(f"Task not found: {name=} with task_key {task_key=}")
 
-        return factory
-
-    def __setitem__(self, name: str, factory: EvalFactory) -> None:
-        task_key = self._task_key(name)
+    def add(self, factory: EvalFactory) -> None:
+        """Register a factory under the key derived from its ``id()``."""
+        task_key = self._task_key(factory.id())
         if task_key in self._registry:
             raise ValueError(f"Cannot register duplicate task with key: {task_key}")
 
-        self._registry[task_key] = (name, factory)
+        self._registry[task_key] = factory
 
     def register(self, task: type[BaseTask]) -> str:
         """The class name is used as the task name."""
         if not issubclass(task, BaseTask):
             raise ValueError(f"Can only register subclasses of BaseTask, got {task}")
-        name = task.__name__
-        self[name] = _Eager(task)
-        return name
+        factory = _Eager(task)
+        self.add(factory)
+        return factory.id()
 
     def register_lazy(self, class_path: str, /) -> None:
         """Register a task by its dotted class path, without importing its module."""
@@ -291,7 +296,7 @@ class Registry:
                 "`eval_framework.tasks.benchmarks.mmlu.MMLU`): "
             )
         base_module, class_name = class_path.rsplit(".", maxsplit=1)
-        self[class_name] = _Lazy(class_name=class_name, module=base_module)
+        self.add(_Lazy(class_name=class_name, module=base_module))
 
 
 _REGISTRY = Registry()
@@ -335,6 +340,7 @@ def register_task(task: type[BaseTask]) -> str:
     return registry().register(task)
 
 
-def register_lazy_task(class_path: str, /) -> None:
+def register_lazy_task(class_path: str, /, registry: Registry | None = None) -> None:
     """Register a task by its dotted class path, without importing its module."""
-    registry().register_lazy(class_path)
+    r = registry if registry is not None else _REGISTRY
+    r.register_lazy(class_path)

eval_framework-0.9.2/src/eval_framework/tasks/task_names.py

@@ -0,0 +1,213 @@
+from enum import Enum
+
+from eval_framework.tasks.base import BaseTask
+from eval_framework.tasks.registry import Registry, register_lazy_task
+from eval_framework.tasks.registry import registry as global_registry
+
+
+class TaskNameEnum(Enum):
+    @property
+    def value(self) -> type[BaseTask]:
+        return super().value
+
+
+def register_all_tasks(registry: Registry | None = None) -> None:
+    """Register all the benchmark tasks with the eval framework
+
+    Uses global registry by default.
+    """
+    registry = registry if registry is not None else global_registry()
+
+    register_math_reasoning_tasks(registry=registry)
+    register_arc_tasks(registry=registry)
+    register_arc_de_tasks(registry=registry)
+    register_bigcodebench_tasks(registry=registry)
+    register_copa_tasks(registry=registry)
+    register_goldenswag_tasks(registry=registry)
+    register_gpqa_tasks(registry=registry)
+    register_gsm8k_tasks(registry=registry)
+    register_hellaswag_tasks(registry=registry)
+    register_humaneval_tasks(registry=registry)
+    register_ifeval_tasks(registry=registry)
+    register_multipl_e_tasks(registry=registry)
+    register_mbpp_tasks(registry=registry)
+    register_mmlu_tasks(registry=registry)
+    register_mmlu_pro_tasks(registry=registry)
+    register_global_mmlu_tasks(registry=registry)
+    register_piqa_tasks(registry=registry)
+    register_sciq_tasks(registry=registry)
+    register_squad_tasks(registry=registry)
+    register_winogrande_tasks(registry=registry)
+    register_csqa_tasks(registry=registry)
+    register_drop_tasks(registry=registry)
+    register_naturalqs_open_tasks(registry=registry)
+    register_social_iqa_tasks(registry=registry)
+    register_medqa_tasks(registry=registry)
+
+
+def register_arc_tasks(registry: Registry) -> None:
+    """Register arc benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.arc.ARC", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.arc.ARC_IDK", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.arc.ARC_OLMES", registry=registry)
+
+
+def register_hellaswag_tasks(registry: Registry) -> None:
+    """Register hellaswag benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.hellaswag.HELLASWAG", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.hellaswag.HELLASWAG_OLMES", registry=registry)
+
+
+def register_piqa_tasks(registry: Registry) -> None:
+    """Register piqa benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.piqa.PIQA", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.piqa.PIQA_IDK", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.piqa.PIQA_OLMES", registry=registry)
+
+
+def register_gpqa_tasks(registry: Registry) -> None:
+    """Register gpqa benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.gpqa.GPQA_OLMES", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.gpqa.GPQA_DIAMOND_COT", registry=registry)
+
+
+def register_gsm8k_tasks(registry: Registry) -> None:
+    """Register gsm8k benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.gsm8k.GSM8K_OLMES", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.gsm8k.GSM8KBPB", registry=registry)
+
+
+def register_math_reasoning_tasks(registry: Registry) -> None:
+    """Register math_reasoning benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.AIME2024", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.AIME2026", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.AIME2025", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.MATHMinervaBPB", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.GSM8KReasoning", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.MATH500", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.MATHMinerva_OLMES", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.MATHMinerva_OLMES_NONL", registry=registry)
+
+
+def register_mmlu_tasks(registry: Registry) -> None:
+    """Register mmlu benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU_IDK", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU_OLMES", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mmlu.FullTextMMLU", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU_COT", registry=registry)
+
+
+def register_humaneval_tasks(registry: Registry) -> None:
+    """Register humaneval benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.humaneval.HumanEvalBPB", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.humaneval.HumanEval_OLMES", registry=registry)
+
+
+def register_mbpp_tasks(registry: Registry) -> None:
+    """Register mbpp benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.mbpp.MBPPBPB", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mbpp.MBPP_OLMES", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mbpp.MBPP_EvalPlus", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mbpp.MBPP_BPB_EvalPlus", registry=registry)
+
+
+def register_bigcodebench_tasks(registry: Registry) -> None:
+    """Register bigcodebench benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.bigcodebench.BigCodeBench_OLMES", registry=registry)
+
+
+def register_arc_de_tasks(registry: Registry) -> None:
+    """Register arc_de benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.arc_de.ARC_DE", registry=registry)
+
+
+def register_copa_tasks(registry: Registry) -> None:
+    """Register copa benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.copa.COPA_OLMES", registry=registry)
+
+
+def register_goldenswag_tasks(registry: Registry) -> None:
+    """Register goldenswag benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.goldenswag.GOLDENSWAG", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.goldenswag.GOLDENSWAG_IDK", registry=registry)
+
+
+def register_ifeval_tasks(registry: Registry) -> None:
+    """Register ifeval benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.ifeval.IFEval", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.ifeval.IFEvalDe", registry=registry)
+
+
+def register_multipl_e_tasks(registry: Registry) -> None:
+    """Register multipl_e benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalCpp", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalJava", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalJs", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalPhp", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalRs", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalSh", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEMBPPCpp", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEMBPPJava", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEMBPPJs", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEMBPPPhp", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEMBPPRs", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEMBPPSh", registry=registry)
+
+
+def register_mmlu_pro_tasks(registry: Registry) -> None:
+    """Register mmlu_pro benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO_IDK", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO_OLMES", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO_COT", registry=registry)
+
+
+def register_global_mmlu_tasks(registry: Registry) -> None:
+    """Register global_mmlu benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.global_mmlu.GlobalMMLU", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.global_mmlu.GlobalMMLU_German", registry=registry)
+
+
+def register_sciq_tasks(registry: Registry) -> None:
+    """Register sciq benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.sciq.SCIQ_OLMES", registry=registry)
+
+
+def register_squad_tasks(registry: Registry) -> None:
+    """Register squad benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.squad.SQuAD_OLMES", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.squad.SQuAD2_MA", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.squad.SQuAD2_MA_NO_SYSPROMPT", registry=registry)
+
+
+def register_winogrande_tasks(registry: Registry) -> None:
+    """Register winogrande benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.winogrande.WINOGRANDECloze", registry=registry)
+
+
+def register_csqa_tasks(registry: Registry) -> None:
+    """Register csqa benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.csqa.CommonsenseQAMC_OLMES", registry=registry)
+
+
+def register_drop_tasks(registry: Registry) -> None:
+    """Register drop benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.drop.DropCompletion_OLMES", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.drop.DropMC_OLMES", registry=registry)
+
+
+def register_naturalqs_open_tasks(registry: Registry) -> None:
+    """Register naturalqs_open benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.naturalqs_open.NaturalQsOpen", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.naturalqs_open.NaturalQsOpenMC_OLMES", registry=registry)
+
+
+def register_social_iqa_tasks(registry: Registry) -> None:
+    """Register social_iqa benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.social_iqa.SocialIQAMC_OLMES", registry=registry)
+
+
+def register_medqa_tasks(registry: Registry) -> None:
+    """Register medqa benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.medqa.MedQAMC_OLMES", registry=registry)