eval-framework 0.8.11__tar.gz → 0.9.1__tar.gz

{eval_framework-0.8.11 → eval_framework-0.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.8.11
+Version: 0.9.1
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -226,7 +226,7 @@ Requires-Dist: jsonschema>=4.26.0,<5
 Requires-Dist: mysql-connector-python>=26.7.0,<27
 Requires-Dist: psycopg2-binary>=2.9.12,<3
 Requires-Dist: sympy>=1.14.0,<2
-Requires-Dist: llm-sandbox[docker]==0.3.39
+Requires-Dist: llm-sandbox[docker]==0.3.44
 Requires-Dist: jsonlines>=4,<5
 Requires-Dist: lxml>=6.1.1,<7
 Requires-Dist: python-iso639>=2026.7.23
@@ -240,7 +240,7 @@ Requires-Dist: eval-framework[determined,api,openai,transformers,accelerate,opti
 Requires-Dist: aleph-alpha-client>=11.5.1 ; extra == 'api'
 Requires-Dist: determined>=0.38.1,<0.39 ; extra == 'determined'
 Requires-Dist: tensorboard==2.21.0 ; extra == 'determined'
-Requires-Dist: openai>=2.52.0,<3 ; extra == 'openai'
+Requires-Dist: openai>=2.53.0,<3 ; extra == 'openai'
 Requires-Dist: tiktoken>=0.13.0,<1 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'optional'
@@ -450,11 +450,13 @@ from eval_framework.main import main
 from eval_framework.tasks.eval_config import EvalConfig
 from template_formatting.formatter import HFFormatter
 
+
 # Define your model
 class MyHuggingFaceModel(HFLLM):
     LLM_NAME = "microsoft/DialoGPT-medium"
     DEFAULT_FORMATTER = partial(HFFormatter, "microsoft/DialoGPT-medium")
 
+
 if __name__ == "__main__":
     # Initialize your model
     llm = MyHuggingFaceModel()

{eval_framework-0.8.11 → eval_framework-0.9.1}/README.md

@@ -189,11 +189,13 @@ from eval_framework.main import main
 from eval_framework.tasks.eval_config import EvalConfig
 from template_formatting.formatter import HFFormatter
 
+
 # Define your model
 class MyHuggingFaceModel(HFLLM):
     LLM_NAME = "microsoft/DialoGPT-medium"
     DEFAULT_FORMATTER = partial(HFFormatter, "microsoft/DialoGPT-medium")
 
+
 if __name__ == "__main__":
     # Initialize your model
     llm = MyHuggingFaceModel()

{eval_framework-0.8.11 → eval_framework-0.9.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.8.11"
+version = "0.9.1"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -33,7 +33,7 @@ dependencies = [
   "mysql-connector-python>=26.7.0,<27", # required for sql-related tasks
   "psycopg2-binary>=2.9.12,<3", # required for sql-related tasks
   "sympy>=1.14.0,<2",
-  "llm-sandbox[docker]==0.3.39",
+  "llm-sandbox[docker]==0.3.44",
   "jsonlines>=4,<5",
   "lxml>=6.1.1,<7",
   "python-iso639>=2026.7.23",
@@ -54,7 +54,7 @@ determined = [
 ]
 api = ["aleph-alpha-client>=11.5.1"]
 openai = [
-  "openai>=2.52.0,<3",
+  "openai>=2.53.0,<3",
   "tiktoken>=0.13.0,<1",
   "transformers>=4.45.2,<5",
 ]

{eval_framework-0.8.11 → eval_framework-0.9.1}/src/eval_framework/response_generator.py

@@ -179,7 +179,7 @@ class ResponseGenerator:
         :param should_preempt_callable: function to check if preempt is called
         :return: list of responses, preempted
         """
-        logger.info(f"{RED}[ Running task {self.task.NAME} against model ------------ ]{RESET}")
+        logger.info(f"{RED}[ Running task {self.task.display_name()} against model ------------ ]{RESET}")
         self.start_time, monotonic_start = time.time(), time.monotonic()
         run_fn = self._generative_output_type_selector()
         self._verify_loaded_metadata_compatibility()

{eval_framework-0.8.11 → eval_framework-0.9.1}/src/eval_framework/tasks/base.py

@@ -2,7 +2,7 @@ import logging
 import os
 import random
 import traceback
-from abc import ABC
+from abc import ABC, abstractmethod
 from collections.abc import Iterable, Sequence
 from enum import Enum
 from pathlib import Path
@@ -86,7 +86,37 @@ SubjectType = TypeVar("SubjectType")
 logger = logging.getLogger(__name__)
 
 
-class BaseTask[SubjectType](ABC):
+class Task(ABC):
+    """The contract a caller relies on to run an evaluation"""
+
+    @abstractmethod
+    def iterate_samples(self, num_samples: int | None = None) -> Iterable[Sample]: ...
+
+    @abstractmethod
+    def generate_completions(
+        self,
+        llm: "BaseLLM",
+        samples: list[Sample],
+        stop_sequences: list[str] | None = None,
+        max_tokens: int | None = None,
+        fail_on_error: bool = True,
+    ) -> list[Completion]:
+        """Run ``llm`` over ``samples`` and return their completions."""
+
+    @abstractmethod
+    def get_metadata(self) -> dict[str, str | list[str]]:
+        """Descriptive metadata about the eval for result reporting."""
+
+    @abstractmethod
+    def get_response_type(self) -> ResponseType: ...
+
+    @abstractmethod
+    def display_name(self) -> str:
+        """Human-readable display name. Is allowed to have special characters and whitespaces."""
+        ...
+
+
+class BaseTask[SubjectType](Task):
     NAME: str
     DATASET_PATH: str
     SAMPLE_SPLIT: str
@@ -160,11 +190,27 @@ class BaseTask[SubjectType](ABC):
 
         assert hasattr(self, "SUBJECTS") and len(self.SUBJECTS) > 0
         if isinstance(self.SUBJECTS[0], tuple):
-            # subjects are specified as strings but we need tuples
-            filters = [tuple(item.strip() for item in subject.split(",")) for subject in custom_subjects]
+            # subjects are specified as comma-separated strings but tuple positions may each hold a
+            # different type (e.g. tuple[str, int, str]). Infer the expected type per position from an
+            # actual subject and cast each part back to it, so it compares equal to the real values
+            # below instead of just their str() form. "*" is a wildcard sentinel and stays a string.
+            num_items = len(self.SUBJECTS[0])
+            position_types = [type(self.SUBJECTS[0][i]) for i in range(num_items)]
+
+            def cast(raw: str, i: int) -> Any:
+                raw = raw.strip()
+                return raw if raw == "*" else position_types[i](raw)
+
+            filters = []
+            for custom_subject in custom_subjects:
+                parts = custom_subject.split(",")
+                assert len(parts) == num_items, (
+                    f"Subject '{custom_subject}' has {len(parts)} parts, expected {num_items} for "
+                    f"task {self.display_name()}"
+                )
+                filters.append(tuple(cast(part, i) for i, part in enumerate(parts)))
 
             # check if all parts of custom subjects exists (* is a wildcard)
-            num_items = len(self.SUBJECTS[0])
             legal_values = [
                 set([s[i] for s in self.SUBJECTS if isinstance(s, tuple)] + ["*"]) for i in range(num_items)
             ]
@@ -174,18 +220,18 @@ class BaseTask[SubjectType](ABC):
                     assert v in legal_values[i], f"Subject part {v} not found in task {self.__class__.__name__}"
 
             # filter task subjects. * is a supported wildcard for a specific item in a tuple, e.g. "DE_DE, *"
-            chosen_subjects = []
+            chosen_subjects: list[tuple] = []
             for subject in self.SUBJECTS:
                 subject_tuple = subject if isinstance(subject, tuple) else tuple(str(subject).split(","))
                 for filter in filters:
                     if all(filter[i] == "*" or filter[i] == subject_tuple[i] for i in range(num_items)):
                         chosen_subjects.append(subject_tuple)
                         break
-            return chosen_subjects  # type: ignore[return-value]
+            return chosen_subjects
         else:
             for cs in custom_subjects:
                 assert cs in self.SUBJECTS, f"Subject {cs} not found in task {self.__class__.__name__}"
-            return custom_subjects  # type: ignore[return-value]
+            return custom_subjects
 
     def _load_hf_dataset(self, **kwargs: Any) -> Any:
         cache_dir: str = os.environ.get("HF_DATASET_CACHE_DIR", f"{Path.home()}/.cache/huggingface/datasets")
@@ -497,3 +543,6 @@ class BaseTask[SubjectType](ABC):
     def METRICS(cls) -> list[type["BaseMetric"]]:
         """For backwards compatibility."""
         return cls.get_metrics()
+
+    def display_name(self) -> str:
+        return self.NAME

{eval_framework-0.8.11 → eval_framework-0.9.1}/src/eval_framework/tasks/benchmarks/gpqa.py

@@ -232,3 +232,8 @@ class GPQA_COT(GPQA):
         # index 1 selects the letter
         answer_key = choices[correct_answer_position][1]
         return answer_key
+
+
+class GPQA_DIAMOND_COT(GPQA_COT):
+    NAME = "GPQA_DIAMOND_COT"
+    SUBJECTS = ["gpqa_diamond"]

{eval_framework-0.8.11 → eval_framework-0.9.1}/src/eval_framework/tasks/markdown_doc.py

@@ -47,7 +47,7 @@ def markdown_doc(
     buf.write(f"- Module: `{module}`\n\n")
 
     if http_path:
-        buf.write(f"- Link to dataset: [{http_path}]({http_path})\n\n")
+        buf.write(f"- Link to dataset: [{http_path}]({http_path})\n")
     else:
         assert example_messages is not None, "a task without a dataset link must supply an example sample"
         for split, size in (split_sizes or {}).items():

{eval_framework-0.8.11 → eval_framework-0.9.1}/src/eval_framework/tasks/registry.py

@@ -18,7 +18,6 @@ __all__ = [
     "EvalFactory",
     "Registry",
     "with_registry",
-    "get_task",
     "is_registered",
     "validate_task_name",
     "registered_task_names",
@@ -36,8 +35,8 @@ class EvalFactory(ABC):
     """
 
     @abstractmethod
-    def task_class(self) -> type[BaseTask]:
-        """Return the task class, importing it on first access if necessary."""
+    def id(self) -> str:
+        "Canonical key used to register this benchmark"
 
     @property
     @abstractmethod
@@ -75,13 +74,10 @@ class EvalFactory(ABC):
         user_prompt_suffix: str | None = None,
     ) -> BaseTask: ...
 
+    @abstractmethod
     def markdown_doc(self, formatters: Sequence[BaseFormatter]) -> str:
         """Render the eval's documentation as markdown."""
-        try:
-            task = self.create(num_fewshot=1, custom_subjects=None, custom_hf_revision=None)
-        except (TypeError, ValueError, AssertionError):
-            task = self.create(num_fewshot=0, custom_subjects=None, custom_hf_revision=None)
-        return task.markdown_doc(formatters)
+        ...
 
 
 class _Lazy(EvalFactory):
@@ -104,6 +100,9 @@ class _Lazy(EvalFactory):
     def source_module(self) -> str:
         return self._module
 
+    def id(self) -> str:
+        return self._class_name
+
     def task_class(self) -> type[BaseTask]:
         if self._loaded is None:
             module = importlib.import_module(self._module)
@@ -152,6 +151,13 @@ class _Lazy(EvalFactory):
         """The eval's human-readable display name (the task's ``NAME``)."""
         return self.task_class().NAME
 
+    def markdown_doc(self, formatters: Sequence[BaseFormatter]) -> str:
+        try:
+            task = self.create(num_fewshot=1, custom_subjects=None, custom_hf_revision=None)
+        except (TypeError, ValueError, AssertionError):
+            task = self.create(num_fewshot=0, custom_subjects=None, custom_hf_revision=None)
+        return task.markdown_doc(formatters)
+
 
 class _Eager(EvalFactory):
     """Wraps an already-imported task class."""
@@ -163,8 +169,8 @@ class _Eager(EvalFactory):
     def source_module(self) -> str:
         return self._task.__module__
 
-    def task_class(self) -> type[BaseTask]:
-        return self._task
+    def id(self) -> str:
+        return self._task.__name__
 
     def create(
         self,
@@ -173,7 +179,7 @@ class _Eager(EvalFactory):
         custom_hf_revision: str | None,
         user_prompt_suffix: str | None = None,
     ) -> BaseTask:
-        return self.task_class().with_overwrite(
+        return self._task.with_overwrite(
             num_fewshot=num_fewshot,
             custom_subjects=custom_subjects,
             custom_hf_revision=custom_hf_revision,
@@ -188,7 +194,7 @@ class _Eager(EvalFactory):
         custom_hf_revision: str | None,
         user_prompt_suffix: str | None = None,
     ) -> BaseTask:
-        perturbation_task_class = create_perturbation_class(self.task_class(), perturbation_config)
+        perturbation_task_class = create_perturbation_class(self._task, perturbation_config)
         return perturbation_task_class.with_overwrite(
             num_fewshot=num_fewshot,
             custom_subjects=custom_subjects,
@@ -198,32 +204,34 @@ class _Eager(EvalFactory):
 
     def response_type(self) -> ResponseType:
         """The eval's response type"""
-        return self.task_class().get_response_type()
+        return self._task.get_response_type()
 
     def metrics(self) -> list[type["BaseMetric"]]:
         """The eval's metrics"""
-        return self.task_class().get_metrics()
+        return self._task.get_metrics()
 
     def display_name(self) -> str:
         """The eval's human-readable display name (the task's ``NAME``)."""
-        return self.task_class().NAME
+        return self._task.NAME
 
+    def markdown_doc(self, formatters: Sequence[BaseFormatter]) -> str:
+        try:
+            task = self.create(num_fewshot=1, custom_subjects=None, custom_hf_revision=None)
+        except (TypeError, ValueError, AssertionError):
+            task = self.create(num_fewshot=0, custom_subjects=None, custom_hf_revision=None)
+        return task.markdown_doc(formatters)
 
-class Registry:
-    """A registry for tasks with support for lazy loading.
 
-    Task names are hashed based on the upper-case name, to avoid issues with
-    ambiguous naming.
-    """
+class Registry:
+    """A registry for Tasks"""
 
     def __init__(self) -> None:
-        # TODO: Lookup only with upper names
-        self._registry: dict[str, tuple[str, EvalFactory]] = dict()
+        self._registry: dict[str, EvalFactory] = dict()
 
     def __iter__(self) -> Iterator[str]:
         """Iterate over all task names in the registry."""
-        for name, _ in self._registry.values():
-            yield name
+        for factory in self._registry.values():
+            yield factory.id()
 
     def task_names(self) -> list[str]:
         """The names of all registered tasks."""
@@ -231,7 +239,8 @@ class Registry:
 
     def items(self) -> Iterator[tuple[str, EvalFactory]]:
         """Iterate over `(task name, EvalFactory)` pairs in the registry."""
-        yield from self._registry.values()
+        for factory in self._registry.values():
+            yield factory.id(), factory
 
     @staticmethod
     def _task_key(name: str, /) -> str:
@@ -249,26 +258,25 @@ class Registry:
     def __getitem__(self, name: str, /) -> EvalFactory:
         task_key = self._task_key(name)
         try:
-            _, factory = self._registry[task_key]
+            return self._registry[task_key]
         except KeyError:
             raise KeyError(f"Task not found: {name=} with task_key {task_key=}")
 
-        return factory
-
-    def __setitem__(self, name: str, factory: EvalFactory) -> None:
-        task_key = self._task_key(name)
+    def add(self, factory: EvalFactory) -> None:
+        """Register a factory under the key derived from its ``id()``."""
+        task_key = self._task_key(factory.id())
         if task_key in self._registry:
             raise ValueError(f"Cannot register duplicate task with key: {task_key}")
 
-        self._registry[task_key] = (name, factory)
+        self._registry[task_key] = factory
 
     def register(self, task: type[BaseTask]) -> str:
         """The class name is used as the task name."""
         if not issubclass(task, BaseTask):
             raise ValueError(f"Can only register subclasses of BaseTask, got {task}")
-        name = task.__name__
-        self[name] = _Eager(task)
-        return name
+        factory = _Eager(task)
+        self.add(factory)
+        return factory.id()
 
     def register_lazy(self, class_path: str, /) -> None:
         """Register a task by its dotted class path, without importing its module."""
@@ -278,7 +286,7 @@ class Registry:
                 "`eval_framework.tasks.benchmarks.mmlu.MMLU`): "
             )
         base_module, class_name = class_path.rsplit(".", maxsplit=1)
-        self[class_name] = _Lazy(class_name=class_name, module=base_module)
+        self.add(_Lazy(class_name=class_name, module=base_module))
 
 
 _REGISTRY = Registry()
@@ -317,19 +325,12 @@ def validate_task_name(name: str) -> str:
     return name
 
 
-def get_task(name: str, /) -> type[BaseTask]:
-    """Return a registered task for a given name.
-
-    Note: This method will import any lazily registered task.
-    """
-    return _REGISTRY[name].task_class()
-
-
 def register_task(task: type[BaseTask]) -> str:
     """The class name is used as the task name."""
     return registry().register(task)
 
 
-def register_lazy_task(class_path: str, /) -> None:
+def register_lazy_task(class_path: str, /, registry: Registry | None = None) -> None:
     """Register a task by its dotted class path, without importing its module."""
-    registry().register_lazy(class_path)
+    r = registry if registry is not None else _REGISTRY
+    r.register_lazy(class_path)

eval_framework-0.9.1/src/eval_framework/tasks/task_names.py

@@ -0,0 +1,213 @@
+from enum import Enum
+
+from eval_framework.tasks.base import BaseTask
+from eval_framework.tasks.registry import Registry, register_lazy_task
+from eval_framework.tasks.registry import registry as global_registry
+
+
+class TaskNameEnum(Enum):
+    @property
+    def value(self) -> type[BaseTask]:
+        return super().value
+
+
+def register_all_tasks(registry: Registry | None = None) -> None:
+    """Register all the benchmark tasks with the eval framework
+
+    Uses global registry by default.
+    """
+    registry = registry if registry is not None else global_registry()
+
+    register_math_reasoning_tasks(registry=registry)
+    register_arc_tasks(registry=registry)
+    register_arc_de_tasks(registry=registry)
+    register_bigcodebench_tasks(registry=registry)
+    register_copa_tasks(registry=registry)
+    register_goldenswag_tasks(registry=registry)
+    register_gpqa_tasks(registry=registry)
+    register_gsm8k_tasks(registry=registry)
+    register_hellaswag_tasks(registry=registry)
+    register_humaneval_tasks(registry=registry)
+    register_ifeval_tasks(registry=registry)
+    register_multipl_e_tasks(registry=registry)
+    register_mbpp_tasks(registry=registry)
+    register_mmlu_tasks(registry=registry)
+    register_mmlu_pro_tasks(registry=registry)
+    register_global_mmlu_tasks(registry=registry)
+    register_piqa_tasks(registry=registry)
+    register_sciq_tasks(registry=registry)
+    register_squad_tasks(registry=registry)
+    register_winogrande_tasks(registry=registry)
+    register_csqa_tasks(registry=registry)
+    register_drop_tasks(registry=registry)
+    register_naturalqs_open_tasks(registry=registry)
+    register_social_iqa_tasks(registry=registry)
+    register_medqa_tasks(registry=registry)
+
+
+def register_arc_tasks(registry: Registry) -> None:
+    """Register arc benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.arc.ARC", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.arc.ARC_IDK", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.arc.ARC_OLMES", registry=registry)
+
+
+def register_hellaswag_tasks(registry: Registry) -> None:
+    """Register hellaswag benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.hellaswag.HELLASWAG", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.hellaswag.HELLASWAG_OLMES", registry=registry)
+
+
+def register_piqa_tasks(registry: Registry) -> None:
+    """Register piqa benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.piqa.PIQA", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.piqa.PIQA_IDK", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.piqa.PIQA_OLMES", registry=registry)
+
+
+def register_gpqa_tasks(registry: Registry) -> None:
+    """Register gpqa benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.gpqa.GPQA_OLMES", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.gpqa.GPQA_DIAMOND_COT", registry=registry)
+
+
+def register_gsm8k_tasks(registry: Registry) -> None:
+    """Register gsm8k benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.gsm8k.GSM8K_OLMES", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.gsm8k.GSM8KBPB", registry=registry)
+
+
+def register_math_reasoning_tasks(registry: Registry) -> None:
+    """Register math_reasoning benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.AIME2024", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.AIME2026", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.AIME2025", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.MATHMinervaBPB", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.GSM8KReasoning", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.MATH500", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.MATHMinerva_OLMES", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.MATHMinerva_OLMES_NONL", registry=registry)
+
+
+def register_mmlu_tasks(registry: Registry) -> None:
+    """Register mmlu benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU_IDK", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU_OLMES", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mmlu.FullTextMMLU", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU_COT", registry=registry)
+
+
+def register_humaneval_tasks(registry: Registry) -> None:
+    """Register humaneval benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.humaneval.HumanEvalBPB", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.humaneval.HumanEval_OLMES", registry=registry)
+
+
+def register_mbpp_tasks(registry: Registry) -> None:
+    """Register mbpp benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.mbpp.MBPPBPB", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mbpp.MBPP_OLMES", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mbpp.MBPP_EvalPlus", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mbpp.MBPP_BPB_EvalPlus", registry=registry)
+
+
+def register_bigcodebench_tasks(registry: Registry) -> None:
+    """Register bigcodebench benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.bigcodebench.BigCodeBench_OLMES", registry=registry)
+
+
+def register_arc_de_tasks(registry: Registry) -> None:
+    """Register arc_de benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.arc_de.ARC_DE", registry=registry)
+
+
+def register_copa_tasks(registry: Registry) -> None:
+    """Register copa benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.copa.COPA_OLMES", registry=registry)
+
+
+def register_goldenswag_tasks(registry: Registry) -> None:
+    """Register goldenswag benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.goldenswag.GOLDENSWAG", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.goldenswag.GOLDENSWAG_IDK", registry=registry)
+
+
+def register_ifeval_tasks(registry: Registry) -> None:
+    """Register ifeval benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.ifeval.IFEval", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.ifeval.IFEvalDe", registry=registry)
+
+
+def register_multipl_e_tasks(registry: Registry) -> None:
+    """Register multipl_e benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalCpp", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalJava", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalJs", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalPhp", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalRs", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalSh", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEMBPPCpp", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEMBPPJava", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEMBPPJs", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEMBPPPhp", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEMBPPRs", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEMBPPSh", registry=registry)
+
+
+def register_mmlu_pro_tasks(registry: Registry) -> None:
+    """Register mmlu_pro benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO_IDK", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO_OLMES", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO_COT", registry=registry)
+
+
+def register_global_mmlu_tasks(registry: Registry) -> None:
+    """Register global_mmlu benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.global_mmlu.GlobalMMLU", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.global_mmlu.GlobalMMLU_German", registry=registry)
+
+
+def register_sciq_tasks(registry: Registry) -> None:
+    """Register sciq benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.sciq.SCIQ_OLMES", registry=registry)
+
+
+def register_squad_tasks(registry: Registry) -> None:
+    """Register squad benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.squad.SQuAD_OLMES", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.squad.SQuAD2_MA", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.squad.SQuAD2_MA_NO_SYSPROMPT", registry=registry)
+
+
+def register_winogrande_tasks(registry: Registry) -> None:
+    """Register winogrande benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.winogrande.WINOGRANDECloze", registry=registry)
+
+
+def register_csqa_tasks(registry: Registry) -> None:
+    """Register csqa benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.csqa.CommonsenseQAMC_OLMES", registry=registry)
+
+
+def register_drop_tasks(registry: Registry) -> None:
+    """Register drop benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.drop.DropCompletion_OLMES", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.drop.DropMC_OLMES", registry=registry)
+
+
+def register_naturalqs_open_tasks(registry: Registry) -> None:
+    """Register naturalqs_open benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.naturalqs_open.NaturalQsOpen", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.naturalqs_open.NaturalQsOpenMC_OLMES", registry=registry)
+
+
+def register_social_iqa_tasks(registry: Registry) -> None:
+    """Register social_iqa benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.social_iqa.SocialIQAMC_OLMES", registry=registry)
+
+
+def register_medqa_tasks(registry: Registry) -> None:
+    """Register medqa benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.medqa.MedQAMC_OLMES", registry=registry)